Agt 60 environmental context improvement

src/__tests__/ConfigService.test.ts

@@ -1,5 +1,5 @@
 import chalk from "chalk";
-import fs from "fs";
+import fs, { truncate } from "fs";
 import path from "path";
 import { ConfigService } from "../services/ConfigService";
 
@@ -23,7 +23,32 @@ describe("ConfigService", () => {
     debug: false,
     options: "temperature=0",
     openRouterApiKey: "test-key",
-    autoScaleAvailableModels: [], // Required by schema
+    appUrl: "https://localhost:8080",
+    appName: "MyApp",
+    autoScaler: true,
+    autoScaleMaxTryPerModel: 2,
+    contextPaths: {
+      includeFilesAndDirectories: false,
+      includeDirectoriesOnly: true,
+    },
+    truncateFilesOnEnvAfterLinesLimit: 1000,
+    discoveryModel: "google/gemini-flash-1.5-8b",
+    strategyModel: "openai/o1-mini",
+    executeModel: "anthropic/claude-3.5-sonnet:beta",
+    autoScaleAvailableModels: [],
+    directoryScanner: {
+      defaultIgnore: ["dist", "coverage", ".next", "build", ".cache", ".husky"],
+      maxDepth: 8,
+      allFiles: true,
+      directoryFirst: true,
+      excludeDirectories: false,
+    },
+    gitDiff: {
+      excludeLockFiles: true,
+      lockFiles: ["package-lock.json"],
+    },
+    referenceExamples: {},
+    timeoutSeconds: 0,
   };
 
   // Helper functions
@@ -70,7 +95,11 @@ describe("ConfigService", () => {
     discoveryModel: "google/gemini-flash-1.5-8b",
     strategyModel: "qwen/qwq-32b-preview",
     executeModel: "anthropic/claude-3.5-sonnet:beta",
-    includeAllFilesOnEnvToContext: false,
+    contextPaths: {
+      includeFilesAndDirectories: false,
+      includeDirectoriesOnly: true,
+    },
+    truncateFilesOnEnvAfterLinesLimit: 1000,
     autoScaleAvailableModels: [
       {
         id: "qwen/qwen-2.5-coder-32b-instruct",
@@ -361,4 +390,64 @@ describe("ConfigService", () => {
       expect(config.packageManager).toBe("bundler");
     });
   });
+
+  describe("environment context configuration", () => {
+    it("should use default values for environment context settings", () => {
+      const minimalConfig = {
+        provider: "open-router",
+        interactive: true,
+        stream: true,
+        debug: false,
+        options: "temperature=0",
+        openRouterApiKey: "test-key",
+        autoScaleAvailableModels: [],
+        contextPaths: {
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
+      };
+
+      (fs.existsSync as jest.Mock).mockReturnValue(true);
+      (fs.readFileSync as jest.Mock).mockReturnValue(
+        JSON.stringify(minimalConfig),
+      );
+
+      const config = configService.getConfig();
+      expect(config.contextPaths.includeFilesAndDirectories).toBe(false);
+      expect(config.contextPaths.includeDirectoriesOnly).toBe(true);
+      expect(config.truncateFilesOnEnvAfterLinesLimit).toBe(1000);
+    });
+
+    it("should allow custom values for environment context settings", () => {
+      const validMockConfig = {
+        provider: "open-router",
+        interactive: true,
+        stream: true,
+        debug: false,
+        options: "temperature=0",
+        openRouterApiKey: "test-key",
+        autoScaleAvailableModels: [],
+      };
+
+      const customConfig = {
+        ...validMockConfig,
+        contextPaths:{
+          includeFilesAndDirectories: true,
+          includeDirectoriesOnly: false,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 500,      
+      };
+
+      (fs.existsSync as jest.Mock).mockReturnValue(true);
+      (fs.readFileSync as jest.Mock).mockReturnValue(
+        JSON.stringify(customConfig),
+      );
+
+      const config = configService.getConfig();
+      expect(config.contextPaths.includeDirectoriesOnly).toBe(false);
+      expect(config.contextPaths.includeFilesAndDirectories).toBe(true);
+      expect(config.truncateFilesOnEnvAfterLinesLimit).toBe(500);
+    });
+  });
 });

src/__tests__/helpers/ConfigServiceTestHelper.ts

@@ -40,7 +40,11 @@ export class ConfigServiceTestHelper {
     discoveryModel: "google/gemini-flash-1.5-8b",
     strategyModel: "qwen/qwq-32b-preview",
     executeModel: "anthropic/claude-3.5-sonnet:beta",
-    includeAllFilesOnEnvToContext: false,
+    contextPaths: {
+      includeFilesAndDirectories: false,
+      includeDirectoriesOnly: true,
+    },
+    truncateFilesOnEnvAfterLinesLimit: 1000,
     autoScaleAvailableModels: [
       {
         id: "qwen/qwen-2.5-coder-32b-instruct",

src/services/ConfigService.ts

@@ -17,7 +17,11 @@ const configSchema = z.object({
   appName: z.string().optional().default("MyApp"),
   autoScaler: z.boolean().optional(),
   autoScaleMaxTryPerModel: z.number().optional(),
-  includeAllFilesOnEnvToContext: z.boolean().optional(),
+  contextPaths: z.object({
+    includeFilesAndDirectories: z.boolean().optional().default(false),
+    includeDirectoriesOnly: z.boolean().optional().default(true),
+  }),
+  truncateFilesOnEnvAfterLinesLimit: z.number().optional().default(1000),
   // Phase-specific model configurations
   discoveryModel: z.string().optional().default("google/gemini-flash-1.5-8b"),
   strategyModel: z.string().optional().default("openai/o1-mini"),
@@ -189,7 +193,11 @@ export class ConfigService {
         discoveryModel: "google/gemini-flash-1.5-8b",
         strategyModel: "qwen/qwq-32b-preview",
         executeModel: "anthropic/claude-3.5-sonnet:beta",
-        includeAllFilesOnEnvToContext: false,
+        contextPaths:{
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
         autoScaleAvailableModels: [
           {
             id: "qwen/qwen-2.5-coder-32b-instruct",

src/services/LLM/LLMContextCreator.ts

@@ -102,13 +102,28 @@ export class LLMContextCreator {
     return baseContext.message;
   }
 
+  private truncateFileContent(
+    content: string | null | undefined,
+    lineLimit: number,
+  ): string {
+    if (!content) return "";
+    const lines = content.split("\n");
+    if (lines.length <= lineLimit) return content;
+    return lines.slice(0, lineLimit).join("\n") + "\n[Content truncated...]";
+  }
+
   private async getEnvironmentDetails(root: string): Promise<string> {
     const scanResult = await this.directoryScanner.scan(root);
     if (!scanResult.success) {
       throw new Error(`Failed to scan directory: ${scanResult.error}`);
     }
 
-    return `# Current Working Directory (${root}) Files\n${scanResult.data}`;
+    const config = this.configService.getConfig();
+    const limit = config.truncateFilesOnEnvAfterLinesLimit;
+
+    const content = String(scanResult.data || "");
+    const truncatedContent = this.truncateFileContent(content, limit);
+    return `# Current Working Directory (${root}) Files\n${truncatedContent}`;
   }
 
   private async getProjectInfo(root: string): Promise<string> {
@@ -187,7 +202,7 @@ ${additionalInstructions ? `${additionalInstructions}` : ""}
     const phaseConfig = this.phaseManager.getCurrentPhaseConfig();
     const customInstructions = await this.loadCustomInstructions();
 
-    const envDetails = config.includeAllFilesOnEnvToContext
+    const envDetails = config.contextPaths.includeFilesAndDirectories
       ? context.environmentDetails
       : "";
 

src/services/LLM/__tests__/PhaseManager.test.ts

@@ -27,7 +27,11 @@ describe("PhaseManager", () => {
     executeModel: "model3",
     autoScaler: false,
     autoScaleMaxTryPerModel: 2,
-    includeAllFilesOnEnvToContext: false,
+    contextPaths:{
+      includeFilesAndDirectories: false,
+      includeDirectoriesOnly: true,
+    },
+    truncateFilesOnEnvAfterLinesLimit: 1000,
     autoScaleAvailableModels: [
       {
         id: "model1",
@@ -123,7 +127,11 @@ describe("PhaseManager", () => {
         appName: "TestApp",
         autoScaler: false,
         autoScaleMaxTryPerModel: 2,
-        includeAllFilesOnEnvToContext: false,
+        contextPaths:{
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
         autoScaleAvailableModels: [
           {
             id: "model1",
@@ -181,7 +189,11 @@ describe("PhaseManager", () => {
         executeModel: "model3",
         autoScaler: false,
         autoScaleMaxTryPerModel: 2,
-        includeAllFilesOnEnvToContext: false,
+        contextPaths:{
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
         autoScaleAvailableModels: [
           {
             id: "model1",

src/services/LLM/tests/LLMContextCreator.test.ts

@@ -83,7 +83,10 @@ describe("LLMContextCreator", () => {
     });
 
     mocker.mockPrototype(ConfigService, "getConfig", {
-      includeAllFilesOnEnvToContext: true,
+      contextPaths: {
+        includeFilesAndDirectories: false,
+        includeDirectoriesOnly: true,
+      },
       customInstructions: "Default custom instructions",
     });
 
@@ -308,7 +311,10 @@ describe("LLMContextCreator", () => {
 
     it("should not include environment details when config flag is false", async () => {
       mocker.mockPrototype(ConfigService, "getConfig", {
-        includeAllFilesOnEnvToContext: false,
+        contextPaths: {
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
         customInstructions: "Default custom instructions",
       });
 
@@ -428,4 +434,56 @@ describe("LLMContextCreator", () => {
       expect(result).toContain("project info");
     });
   });
+
+  describe("file content truncation", () => {
+    it("should truncate file content when it exceeds the limit", async () => {
+      const longContent = Array(1500).fill("line").join("\n");
+      mocker.mockPrototype(DirectoryScanner, "scan", {
+        success: true,
+        data: longContent,
+      });
+
+      mocker.mockPrototype(ConfigService, "getConfig", {
+        contextPaths: {
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
+        customInstructions: "test",
+      });
+
+      const context = await contextCreator.create(
+        "test message",
+        "/root",
+        true,
+      );
+      expect(context).toContain("[Content truncated...]");
+      expect(context.split("\n").length).toBeLessThan(1500);
+    });
+
+    it("should not truncate file content when under the limit", async () => {
+      const shortContent = Array(500).fill("line").join("\n");
+      mocker.mockPrototype(DirectoryScanner, "scan", {
+        success: true,
+        data: shortContent,
+      });
+
+      mocker.mockPrototype(ConfigService, "getConfig", {
+        contextPaths: {
+          includeFilesAndDirectories: false,
+          includeDirectoriesOnly: true,
+        },
+        truncateFilesOnEnvAfterLinesLimit: 1000,
+        customInstructions: "test",
+      });
+
+      const context = await contextCreator.create(
+        "test message",
+        "/root",
+        true,
+      );
+      expect(context).not.toContain("[Content truncated...]");
+      expect(context.split("\n").length).toBeLessThan(1000);
+    });
+  });
 });