Improved error detection BEN-1078

app/api/generate/route.ts

@@ -10,6 +10,28 @@ const benchify = new Benchify({
     apiKey: process.env.BENCHIFY_API_KEY,
 });
 
+const debug = false;
+const buggyCode = [
+    {
+        path: "src/App.tsx",
+        content: `import React from 'react';
+
+const App = () => {
+    const message = "Hello World;  // Missing closing quote
+    const title = 'Welcome to my app';
+
+    return (
+        <div>
+            <h1>{title}</h1>
+            <p>{message}</p>
+        </div>
+    );
+};
+
+export default App;`
+    }
+];
+
 export async function POST(request: NextRequest) {
     try {
         const body = await request.json();
@@ -27,7 +49,12 @@ export async function POST(request: NextRequest) {
         const { description } = validationResult.data;
 
         // Generate the Vue app using OpenAI
-        const generatedFiles = await generateApp(description);
+        let generatedFiles;
+        if (debug) {
+            generatedFiles = buggyCode;
+        } else {
+            generatedFiles = await generateApp(description);
+        }
 
         // Repair the generated code using Benchify's API
         // const { data } = await benchify.fixer.run({

components/ui-builder/preview-card.tsx

@@ -129,8 +129,8 @@ export function PreviewCard({
                                                 </div>
                                                 <div className="flex-1 min-w-0">
                                                     <p className={`text-sm font-medium ${isCompleted ? 'text-green-700 dark:text-green-400' :
-                                                            isCurrent ? 'text-primary' :
-                                                                'text-muted-foreground'
+                                                        isCurrent ? 'text-primary' :
+                                                            'text-muted-foreground'
                                                         }`}>
                                                         {step.label}
                                                     </p>

lib/e2b.ts

@@ -3,6 +3,7 @@ import { benchifyFileSchema } from './schemas';
 import { z } from 'zod';
 import { fetchAllSandboxFiles } from './file-filter';
 import { applyTransformations } from './sandbox-helpers';
+import { detectCodeErrors, parseTypeScriptErrors } from './error-detection';
 
 const E2B_API_KEY = process.env.E2B_API_KEY;
 
@@ -59,8 +60,11 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
                 console.log('New packages installed successfully:', result.stdout);
                 if (result.stderr) {
                     console.warn('npm install warnings:', result.stderr);
-                    // Parse npm install errors
-                    if (result.stderr.includes('npm ERR!')) {
+                    // Only treat critical npm errors as build errors (not warnings or peer dep issues)
+                    if (result.stderr.includes('npm ERR!') &&
+                        (result.stderr.includes('ENOTFOUND') ||
+                            result.stderr.includes('ECONNREFUSED') ||
+                            result.stderr.includes('permission denied'))) {
                         buildErrors.push({
                             type: 'build',
                             message: 'Package installation failed: ' + result.stderr.split('npm ERR!')[1]?.trim()
@@ -79,39 +83,47 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
         }
     }
 
-    // Run TypeScript check to catch type errors
+    // Start the dev server and check logs for errors (let Vite handle error detection)
     try {
-        console.log('Running TypeScript check...');
-        const tscResult = await sandbox.commands.run('cd /app && npx tsc --noEmit --skipLibCheck');
-
-        if (tscResult.exitCode !== 0 && tscResult.stderr) {
-            console.log('TypeScript errors found:', tscResult.stderr);
-            const tsErrors = parseTypeScriptErrors(tscResult.stderr);
-            buildErrors.push(...tsErrors);
+        console.log('Starting dev server...');
+        // Start dev server in background
+        const devServerResult = await sandbox.commands.run('cd /app && npm run dev', { background: true });
+
+        console.log('Dev server command executed');
+        console.log('Dev server exit code:', devServerResult.exitCode);
+        console.log('Dev server stderr:', devServerResult.stderr || 'No stderr');
+        console.log('Dev server stdout:', devServerResult.stdout || 'No stdout');
+
+        // Give it a moment to start and potentially fail
+        console.log('Waiting 3 seconds for dev server to start...');
+        await new Promise(resolve => setTimeout(resolve, 3000));
+
+        // Check the initial output for immediate errors
+        if (devServerResult.stderr || devServerResult.stdout) {
+            const allOutput = (devServerResult.stderr || '') + '\n' + (devServerResult.stdout || '');
+
+            // Use the error detection module
+            const errorResult = detectCodeErrors(allOutput);
+
+            if (errorResult.hasErrors) {
+                console.log('🔴 CODE ERRORS DETECTED!');
+                buildErrors.push(...errorResult.errors);
+            } else if (errorResult.isInfrastructureOnly) {
+                console.log('⚠️  Only infrastructure errors detected (ignoring)');
+            } else {
+                console.log('✅ No errors detected');
+            }
+        } else {
+            console.log('⚠️  No stderr or stdout from dev server command');
         }
-    } catch (error) {
-        console.error('TypeScript check failed:', error);
-        buildErrors.push({
-            type: 'typescript',
-            message: `TypeScript check failed: ${error instanceof Error ? error.message : String(error)}`
-        });
-    }
 
-    // Try to build the project to catch build-time errors
-    try {
-        console.log('Running build check...');
-        const buildResult = await sandbox.commands.run('cd /app && npm run build');
-
-        if (buildResult.exitCode !== 0) {
-            console.log('Build errors found:', buildResult.stderr);
-            const viteErrors = parseViteBuildErrors(buildResult.stderr);
-            buildErrors.push(...viteErrors);
-        }
+        console.log('Dev server started, output checked');
+        console.log('Total build errors found:', buildErrors.length);
     } catch (error) {
-        console.error('Build check failed:', error);
+        console.error('Dev server check failed:', error);
         buildErrors.push({
             type: 'build',
-            message: `Build failed: ${error instanceof Error ? error.message : String(error)}`
+            message: `Dev server failed to start: ${error instanceof Error ? error.message : String(error)}`
         });
     }
 
@@ -130,60 +142,7 @@ export async function createSandbox({ files }: { files: z.infer<typeof benchifyF
     };
 }
 
-function parseTypeScriptErrors(stderr: string): BuildError[] {
-    const errors: BuildError[] = [];
-    const lines = stderr.split('\n');
-
-    for (const line of lines) {
-        // Match TypeScript error pattern: file(line,column): error TS####: message
-        const match = line.match(/(.+)\((\d+),(\d+)\): error TS\d+: (.+)/);
-        if (match) {
-            const [, file, line, column, message] = match;
-            errors.push({
-                type: 'typescript',
-                message: message.trim(),
-                file: file.replace('/app/', ''),
-                line: parseInt(line),
-                column: parseInt(column)
-            });
-        }
-    }
 
-    // If no specific errors found but stderr has content, add generic error
-    if (errors.length === 0 && stderr.trim()) {
-        errors.push({
-            type: 'typescript',
-            message: 'TypeScript compilation failed: ' + stderr.trim()
-        });
-    }
-
-    return errors;
-}
-
-function parseViteBuildErrors(stderr: string): BuildError[] {
-    const errors: BuildError[] = [];
-    const lines = stderr.split('\n');
-
-    for (const line of lines) {
-        // Match Vite build error patterns
-        if (line.includes('error') || line.includes('Error')) {
-            errors.push({
-                type: 'build',
-                message: line.trim()
-            });
-        }
-    }
-
-    // If no specific errors found but stderr has content, add generic error
-    if (errors.length === 0 && stderr.trim()) {
-        errors.push({
-            type: 'build',
-            message: 'Build failed: ' + stderr.trim()
-        });
-    }
-
-    return errors;
-}
 
 function extractNewPackages(packageJsonContent: string): string[] {
     try {

lib/error-detection.ts

@@ -0,0 +1,149 @@
+interface BuildError {
+    type: 'typescript' | 'build' | 'runtime';
+    message: string;
+    file?: string;
+    line?: number;
+    column?: number;
+}
+
+export interface ErrorDetectionResult {
+    hasErrors: boolean;
+    errors: BuildError[];
+    isInfrastructureOnly: boolean;
+}
+
+/**
+ * Detects if output contains code-related errors (not infrastructure issues)
+ */
+export function detectCodeErrors(output: string): ErrorDetectionResult {
+    console.log('=== CHECKING OUTPUT FOR CODE ERRORS ===');
+    console.log('Output length:', output.length);
+    console.log('Full output:', output);
+
+    // Check for actual code errors that users care about
+    const hasSyntaxError = output.includes('SyntaxError');
+    const hasUnexpectedToken = output.includes('Unexpected token');
+    const hasParseError = output.includes('Parse error');
+    const hasUnterminatedString = output.includes('Unterminated string');
+    const hasModuleError = output.includes('Cannot resolve module') || output.includes('Module not found');
+    const hasImportError = output.includes('Cannot resolve import');
+
+    // Check for infrastructure errors that should be ignored
+    const isInfrastructureError = output.includes('EACCES: permission denied') ||
+        output.includes('failed to load config from /app/vite.config.ts') ||
+        output.includes('error when starting dev server') ||
+        output.includes('/app/node_modules/.vite-temp/');
+
+    console.log('Error pattern checks (focusing on code errors):');
+    console.log('- Has "SyntaxError":', hasSyntaxError);
+    console.log('- Has "Unexpected token":', hasUnexpectedToken);
+    console.log('- Has "Parse error":', hasParseError);
+    console.log('- Has "Unterminated string":', hasUnterminatedString);
+    console.log('- Has module/import errors:', hasModuleError || hasImportError);
+    console.log('- Is infrastructure error (ignoring):', isInfrastructureError);
+
+    const hasCodeErrors = hasSyntaxError || hasUnexpectedToken || hasParseError ||
+        hasUnterminatedString || hasModuleError || hasImportError;
+
+    // Only report actual code errors, not infrastructure issues
+    if (hasCodeErrors && !isInfrastructureError) {
+        console.log('🔴 CODE ERRORS DETECTED! Parsing...');
+        const errors = parseErrorsFromOutput(output);
+        console.log('Parsed errors:', errors);
+
+        return {
+            hasErrors: true,
+            errors,
+            isInfrastructureOnly: false
+        };
+    } else if (isInfrastructureError && !hasCodeErrors) {
+        console.log('⚠️  Only infrastructure errors detected (ignoring)');
+        return {
+            hasErrors: false,
+            errors: [],
+            isInfrastructureOnly: true
+        };
+    } else {
+        console.log('✅ No code errors detected');
+        return {
+            hasErrors: false,
+            errors: [],
+            isInfrastructureOnly: false
+        };
+    }
+}
+
+/**
+ * Parses TypeScript compilation errors
+ */
+export function parseTypeScriptErrors(stderr: string): BuildError[] {
+    const errors: BuildError[] = [];
+    const lines = stderr.split('\n');
+
+    for (const line of lines) {
+        // Match TypeScript error pattern: file(line,column): error TS####: message
+        const match = line.match(/(.+)\((\d+),(\d+)\): error TS\d+: (.+)/);
+        if (match) {
+            const [, file, line, column, message] = match;
+            // Filter out common non-critical errors that might be false positives
+            const lowerMessage = message.toLowerCase();
+            if (!lowerMessage.includes('deprecated') &&
+                !lowerMessage.includes('unused') &&
+                !lowerMessage.includes('implicit any')) {
+                errors.push({
+                    type: 'typescript',
+                    message: message.trim(),
+                    file: file.replace('/app/', ''),
+                    line: parseInt(line),
+                    column: parseInt(column)
+                });
+            }
+        }
+    }
+
+    return errors;
+}
+
+/**
+ * Parses errors from build/dev server output
+ */
+function parseErrorsFromOutput(output: string): BuildError[] {
+    console.log('🔍 parseErrorsFromOutput called with input length:', output.length);
+    const errors: BuildError[] = [];
+    const lines = output.split('\n');
+    console.log('Total lines to process:', lines.length);
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        console.log(`Line ${i + 1}: "${line}"`);
+
+        // Only match actual code errors, not infrastructure issues
+        const hasCodeError = line.includes('SyntaxError') ||
+            line.includes('Unexpected token') ||
+            line.includes('Parse error') ||
+            line.includes('Unterminated string') ||
+            line.includes('Cannot resolve module') ||
+            line.includes('Module not found') ||
+            line.includes('Cannot resolve import');
+
+        // Skip infrastructure errors
+        const isInfrastructureError = line.includes('EACCES: permission denied') ||
+            line.includes('failed to load config') ||
+            line.includes('error when starting dev server') ||
+            line.includes('/app/node_modules/.vite-temp/');
+
+        console.log(`  - Has code error: ${hasCodeError}`);
+        console.log(`  - Is infrastructure error (skip): ${isInfrastructureError}`);
+
+        if (hasCodeError && !isInfrastructureError) {
+            console.log(`  ✅ FOUND ERROR: "${line}"`);
+            errors.push({
+                type: 'build',
+                message: line.trim()
+            });
+        }
+    }
+
+    console.log(`🎯 parseErrorsFromOutput found ${errors.length} errors:`, errors);
+    return errors;
+}