Skip to content

Commit 8315d10

Browse files
juancastanojuancastano
authored
Improved error detection BEN-1078 (#28)
### TL;DR Added improved error detection for code generation with debug mode support. ### What changed? - Added a debug mode flag and sample buggy code for testing error detection - Created a new `error-detection.ts` module with specialized functions for detecting and parsing code errors - Improved sandbox error detection by focusing on actual code errors rather than infrastructure issues - Enhanced npm error handling to ignore non-critical warnings - Replaced the TypeScript check and build process with a dev server approach for more accurate error detection ### How to test? 1. Set `debug = true` in `app/api/generate/route.ts` to test with the sample buggy code 2. Generate an app to see the error detection in action 3. Check the console logs for detailed error detection information 4. Verify that infrastructure errors are properly ignored while actual code errors are detected ### Why make this change? The previous error detection system was too strict and would flag infrastructure or non-critical issues as errors. This change improves the user experience by focusing on actual code errors that need fixing, while ignoring harmless warnings or infrastructure-related messages. The addition of debug mode also makes it easier to test and improve the error detection system without generating new code each time.
2 parents 130774d + fe3cf2c commit 8315d10

File tree

4 files changed

+220
-85
lines changed

4 files changed

+220
-85
lines changed

‎app/api/generate/route.ts

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,28 @@ const benchify = new Benchify({
1010
apiKey: process.env.BENCHIFY_API_KEY,
1111
});
1212

13+
const debug = false;
14+
const buggyCode = [
15+
{
16+
path: "src/App.tsx",
17+
content: `import React from 'react';
18+
19+
const App = () => {
20+
const message = "Hello World; // Missing closing quote
21+
const title = 'Welcome to my app';
22+
23+
return (
24+
<div>
25+
<h1>{title}</h1>
26+
<p>{message}</p>
27+
</div>
28+
);
29+
};
30+
31+
export default App;`
32+
}
33+
];
34+
1335
export async function POST(request: NextRequest) {
1436
try {
1537
const body = await request.json();
@@ -27,7 +49,12 @@ export async function POST(request: NextRequest) {
2749
const { description } = validationResult.data;
2850

2951
// Generate the Vue app using OpenAI
30-
const generatedFiles = await generateApp(description);
52+
let generatedFiles;
53+
if (debug) {
54+
generatedFiles = buggyCode;
55+
} else {
56+
generatedFiles = await generateApp(description);
57+
}
3158

3259
// Repair the generated code using Benchify's API
3360
// const { data } = await benchify.fixer.run({

‎components/ui-builder/preview-card.tsx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,8 @@ export function PreviewCard({
129129
</div>
130130
<div className="flex-1 min-w-0">
131131
<p className={`text-sm font-medium ${isCompleted ? 'text-green-700 dark:text-green-400' :
132-
isCurrent ? 'text-primary' :
133-
'text-muted-foreground'
132+
isCurrent ? 'text-primary' :
133+
'text-muted-foreground'
134134
}`}>
135135
{step.label}
136136
</p>

‎lib/e2b.ts

Lines changed: 41 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { benchifyFileSchema } from './schemas';
33
import { z } from 'zod';
44
import { fetchAllSandboxFiles } from './file-filter';
55
import { applyTransformations } from './sandbox-helpers';
6+
import { detectCodeErrors, parseTypeScriptErrors } from './error-detection';
67

78
const E2B_API_KEY = process.env.E2B_API_KEY;
89

@@ -59,8 +60,11 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
5960
console.log('New packages installed successfully:', result.stdout);
6061
if (result.stderr) {
6162
console.warn('npm install warnings:', result.stderr);
62-
// Parse npm install errors
63-
if (result.stderr.includes('npm ERR!')) {
63+
// Only treat critical npm errors as build errors (not warnings or peer dep issues)
64+
if (result.stderr.includes('npm ERR!') &&
65+
(result.stderr.includes('ENOTFOUND') ||
66+
result.stderr.includes('ECONNREFUSED') ||
67+
result.stderr.includes('permission denied'))) {
6468
buildErrors.push({
6569
type: 'build',
6670
message: 'Package installation failed: ' + result.stderr.split('npm ERR!')[1]?.trim()
@@ -79,39 +83,47 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
7983
}
8084
}
8185

82-
// Run TypeScript check to catch type errors
86+
// Start the dev server and check logs for errors (let Vite handle error detection)
8387
try {
84-
console.log('Running TypeScript check...');
85-
const tscResult = await sandbox.commands.run('cd /app && npx tsc --noEmit --skipLibCheck');
86-
87-
if (tscResult.exitCode !== 0 && tscResult.stderr) {
88-
console.log('TypeScript errors found:', tscResult.stderr);
89-
const tsErrors = parseTypeScriptErrors(tscResult.stderr);
90-
buildErrors.push(...tsErrors);
88+
console.log('Starting dev server...');
89+
// Start dev server in background
90+
const devServerResult = await sandbox.commands.run('cd /app && npm run dev', { background: true });
91+
92+
console.log('Dev server command executed');
93+
console.log('Dev server exit code:', devServerResult.exitCode);
94+
console.log('Dev server stderr:', devServerResult.stderr || 'No stderr');
95+
console.log('Dev server stdout:', devServerResult.stdout || 'No stdout');
96+
97+
// Give it a moment to start and potentially fail
98+
console.log('Waiting 3 seconds for dev server to start...');
99+
await new Promise(resolve => setTimeout(resolve, 3000));
100+
101+
// Check the initial output for immediate errors
102+
if (devServerResult.stderr || devServerResult.stdout) {
103+
const allOutput = (devServerResult.stderr || '') + '\n' + (devServerResult.stdout || '');
104+
105+
// Use the error detection module
106+
const errorResult = detectCodeErrors(allOutput);
107+
108+
if (errorResult.hasErrors) {
109+
console.log('🔴 CODE ERRORS DETECTED!');
110+
buildErrors.push(...errorResult.errors);
111+
} else if (errorResult.isInfrastructureOnly) {
112+
console.log('⚠️ Only infrastructure errors detected (ignoring)');
113+
} else {
114+
console.log('✅ No errors detected');
115+
}
116+
} else {
117+
console.log('⚠️ No stderr or stdout from dev server command');
91118
}
92-
} catch (error) {
93-
console.error('TypeScript check failed:', error);
94-
buildErrors.push({
95-
type: 'typescript',
96-
message: `TypeScript check failed: ${error instanceof Error ? error.message : String(error)}`
97-
});
98-
}
99119

100-
// Try to build the project to catch build-time errors
101-
try {
102-
console.log('Running build check...');
103-
const buildResult = await sandbox.commands.run('cd /app && npm run build');
104-
105-
if (buildResult.exitCode !== 0) {
106-
console.log('Build errors found:', buildResult.stderr);
107-
const viteErrors = parseViteBuildErrors(buildResult.stderr);
108-
buildErrors.push(...viteErrors);
109-
}
120+
console.log('Dev server started, output checked');
121+
console.log('Total build errors found:', buildErrors.length);
110122
} catch (error) {
111-
console.error('Build check failed:', error);
123+
console.error('Dev server check failed:', error);
112124
buildErrors.push({
113125
type: 'build',
114-
message: `Build failed: ${error instanceof Error ? error.message : String(error)}`
126+
message: `Dev server failed to start: ${error instanceof Error ? error.message : String(error)}`
115127
});
116128
}
117129

@@ -130,60 +142,7 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
130142
};
131143
}
132144

133-
function parseTypeScriptErrors(stderr: string): BuildError[] {
134-
const errors: BuildError[] = [];
135-
const lines = stderr.split('\n');
136-
137-
for (const line of lines) {
138-
// Match TypeScript error pattern: file(line,column): error TS####: message
139-
const match = line.match(/(.+)\((\d+),(\d+)\): error TS\d+: (.+)/);
140-
if (match) {
141-
const [, file, line, column, message] = match;
142-
errors.push({
143-
type: 'typescript',
144-
message: message.trim(),
145-
file: file.replace('/app/', ''),
146-
line: parseInt(line),
147-
column: parseInt(column)
148-
});
149-
}
150-
}
151145

152-
// If no specific errors found but stderr has content, add generic error
153-
if (errors.length === 0 && stderr.trim()) {
154-
errors.push({
155-
type: 'typescript',
156-
message: 'TypeScript compilation failed: ' + stderr.trim()
157-
});
158-
}
159-
160-
return errors;
161-
}
162-
163-
function parseViteBuildErrors(stderr: string): BuildError[] {
164-
const errors: BuildError[] = [];
165-
const lines = stderr.split('\n');
166-
167-
for (const line of lines) {
168-
// Match Vite build error patterns
169-
if (line.includes('error') || line.includes('Error')) {
170-
errors.push({
171-
type: 'build',
172-
message: line.trim()
173-
});
174-
}
175-
}
176-
177-
// If no specific errors found but stderr has content, add generic error
178-
if (errors.length === 0 && stderr.trim()) {
179-
errors.push({
180-
type: 'build',
181-
message: 'Build failed: ' + stderr.trim()
182-
});
183-
}
184-
185-
return errors;
186-
}
187146

188147
function extractNewPackages(packageJsonContent: string): string[] {
189148
try {

‎lib/error-detection.ts

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
interface BuildError {
2+
type: 'typescript' | 'build' | 'runtime';
3+
message: string;
4+
file?: string;
5+
line?: number;
6+
column?: number;
7+
}
8+
9+
export interface ErrorDetectionResult {
10+
hasErrors: boolean;
11+
errors: BuildError[];
12+
isInfrastructureOnly: boolean;
13+
}
14+
15+
/**
16+
* Detects if output contains code-related errors (not infrastructure issues)
17+
*/
18+
export function detectCodeErrors(output: string): ErrorDetectionResult {
19+
console.log('=== CHECKING OUTPUT FOR CODE ERRORS ===');
20+
console.log('Output length:', output.length);
21+
console.log('Full output:', output);
22+
23+
// Check for actual code errors that users care about
24+
const hasSyntaxError = output.includes('SyntaxError');
25+
const hasUnexpectedToken = output.includes('Unexpected token');
26+
const hasParseError = output.includes('Parse error');
27+
const hasUnterminatedString = output.includes('Unterminated string');
28+
const hasModuleError = output.includes('Cannot resolve module') || output.includes('Module not found');
29+
const hasImportError = output.includes('Cannot resolve import');
30+
31+
// Check for infrastructure errors that should be ignored
32+
const isInfrastructureError = output.includes('EACCES: permission denied') ||
33+
output.includes('failed to load config from /app/vite.config.ts') ||
34+
output.includes('error when starting dev server') ||
35+
output.includes('/app/node_modules/.vite-temp/');
36+
37+
console.log('Error pattern checks (focusing on code errors):');
38+
console.log('- Has "SyntaxError":', hasSyntaxError);
39+
console.log('- Has "Unexpected token":', hasUnexpectedToken);
40+
console.log('- Has "Parse error":', hasParseError);
41+
console.log('- Has "Unterminated string":', hasUnterminatedString);
42+
console.log('- Has module/import errors:', hasModuleError || hasImportError);
43+
console.log('- Is infrastructure error (ignoring):', isInfrastructureError);
44+
45+
const hasCodeErrors = hasSyntaxError || hasUnexpectedToken || hasParseError ||
46+
hasUnterminatedString || hasModuleError || hasImportError;
47+
48+
// Only report actual code errors, not infrastructure issues
49+
if (hasCodeErrors && !isInfrastructureError) {
50+
console.log('🔴 CODE ERRORS DETECTED! Parsing...');
51+
const errors = parseErrorsFromOutput(output);
52+
console.log('Parsed errors:', errors);
53+
54+
return {
55+
hasErrors: true,
56+
errors,
57+
isInfrastructureOnly: false
58+
};
59+
} else if (isInfrastructureError && !hasCodeErrors) {
60+
console.log('⚠️ Only infrastructure errors detected (ignoring)');
61+
return {
62+
hasErrors: false,
63+
errors: [],
64+
isInfrastructureOnly: true
65+
};
66+
} else {
67+
console.log('✅ No code errors detected');
68+
return {
69+
hasErrors: false,
70+
errors: [],
71+
isInfrastructureOnly: false
72+
};
73+
}
74+
}
75+
76+
/**
77+
* Parses TypeScript compilation errors
78+
*/
79+
export function parseTypeScriptErrors(stderr: string): BuildError[] {
80+
const errors: BuildError[] = [];
81+
const lines = stderr.split('\n');
82+
83+
for (const line of lines) {
84+
// Match TypeScript error pattern: file(line,column): error TS####: message
85+
const match = line.match(/(.+)\((\d+),(\d+)\): error TS\d+: (.+)/);
86+
if (match) {
87+
const [, file, line, column, message] = match;
88+
// Filter out common non-critical errors that might be false positives
89+
const lowerMessage = message.toLowerCase();
90+
if (!lowerMessage.includes('deprecated') &&
91+
!lowerMessage.includes('unused') &&
92+
!lowerMessage.includes('implicit any')) {
93+
errors.push({
94+
type: 'typescript',
95+
message: message.trim(),
96+
file: file.replace('/app/', ''),
97+
line: parseInt(line),
98+
column: parseInt(column)
99+
});
100+
}
101+
}
102+
}
103+
104+
return errors;
105+
}
106+
107+
/**
108+
* Parses errors from build/dev server output
109+
*/
110+
function parseErrorsFromOutput(output: string): BuildError[] {
111+
console.log('🔍 parseErrorsFromOutput called with input length:', output.length);
112+
const errors: BuildError[] = [];
113+
const lines = output.split('\n');
114+
console.log('Total lines to process:', lines.length);
115+
116+
for (let i = 0; i < lines.length; i++) {
117+
const line = lines[i];
118+
console.log(`Line ${i + 1}: "${line}"`);
119+
120+
// Only match actual code errors, not infrastructure issues
121+
const hasCodeError = line.includes('SyntaxError') ||
122+
line.includes('Unexpected token') ||
123+
line.includes('Parse error') ||
124+
line.includes('Unterminated string') ||
125+
line.includes('Cannot resolve module') ||
126+
line.includes('Module not found') ||
127+
line.includes('Cannot resolve import');
128+
129+
// Skip infrastructure errors
130+
const isInfrastructureError = line.includes('EACCES: permission denied') ||
131+
line.includes('failed to load config') ||
132+
line.includes('error when starting dev server') ||
133+
line.includes('/app/node_modules/.vite-temp/');
134+
135+
console.log(` - Has code error: ${hasCodeError}`);
136+
console.log(` - Is infrastructure error (skip): ${isInfrastructureError}`);
137+
138+
if (hasCodeError && !isInfrastructureError) {
139+
console.log(` ✅ FOUND ERROR: "${line}"`);
140+
errors.push({
141+
type: 'build',
142+
message: line.trim()
143+
});
144+
}
145+
}
146+
147+
console.log(`🎯 parseErrorsFromOutput found ${errors.length} errors:`, errors);
148+
return errors;
149+
}

0 commit comments

Comments
 (0)