updated main command line with more params + cleaning

Author: islem-esi

ExecutionAgent.sh

@@ -1,51 +1,115 @@
 #!/bin/bash
 
+# Default value for the number parameter
+num=40
+
 # Function to extract project name from GitHub URL
 # Extracts the last component of the URL, which is usually the project name
 extract_project_name() {
   local url="$1"
   echo "$url" | awk -F '/' '{print $(NF)}'
 }
 
+# Function to run the command and handle retries
+run_with_retries() {
+  local command="$1"
+  local project_name="$2"
+  local max_retries=2
+  local attempt=1
+
+  while [[ $attempt -le $max_retries ]]; do
+    echo "======================================================================"
+    echo "STARTING ITERATION $attempt:"
+    echo "PROJECT: $project_name"
+    echo "======================================================================"
+
+    eval "$command"
+    result=$(python3.10 post_process.py "$project_name")
+
+    if [[ "$result" == "SUCCESS" ]]; then
+      echo "Post-process succeeded."
+      return
+    fi
+
+    echo "Attempt $attempt failed with FAILURE. Retrying..."
+    ((attempt++))
+  done
+
+  while true; do
+    echo "======================================================================"
+    echo "PROMPTING USER FOR ADDITIONAL RETRY:"
+    echo "PROJECT: $project_name"
+    echo "======================================================================"
+
+    read -p "Post-process failed after $max_retries attempts. Do you want to retry again? (yes/no): " user_input
+    case "$user_input" in
+      [Yy]* ) eval "$command"; result=$(python3.10 post_process.py "$project_name");
+              if [[ "$result" == "SUCCESS" ]]; then
+                echo "Post-process succeeded."
+                return
+              fi
+              ;;
+      [Nn]* ) echo "Exiting retry loop."; break;;
+      * ) echo "Please answer yes or no.";;
+    esac
+  done
+}
+
+# Parse command-line arguments
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --repo)
+      repo_url="$2"
+      shift 2
+      ;;
+    -l)
+      num="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
 # Set up API key, increment experiment, and prepare AI settings
 python3.10 setup_api_key.py  # Sets up the API key required for the scripts
 python3.10 experimental_setups/increment_experiment.py  # Updates experimental parameters
 python3.10 prepare_ai_settings.py  # Prepares the AI settings configuration
 
 # Check for the --repo argument or file path
-if [[ "$1" == "--repo" ]]; then
+if [[ -n "$repo_url" ]]; then
   # Ensure the user provided a GitHub URL with the --repo argument
-  if [[ -z "$2" ]]; then
+  if [[ -z "$repo_url" ]]; then
     echo "Error: --repo argument requires a GitHub URL."
     echo "Usage: ./script_name.sh --repo <github_repo_url>"
     exit 1
   fi
 
   # Extract the project name from the provided GitHub URL
-  github_url="$2"
-  project_name=$(extract_project_name "$github_url")
+  project_name=$(extract_project_name "$repo_url")
 
   # Call get_main_language.py to determine the main language of the repository
   # The Python script is expected to return a string like "Primary language: <language>"
-  primary_language=$(python3.10 get_main_language.py "$github_url")
+  primary_language=$(python3.10 get_main_language.py "$repo_url")
   echo "$primary_language"
 
   # Continue processing for a single repository
   echo "$project_name"  # Print the project name
-  echo "$github_url"    # Print the GitHub URL
+  echo "$repo_url"      # Print the GitHub URL
 
   # Initialize an empty Docker configuration file
   echo "{}" > ~/.docker/config.json
 
   # Call the Python script to clone the repo and set metadata
-  python3.10 clone_and_set_metadata.py "$project_name" "$github_url" "$primary_language"
+  python3.10 clone_and_set_metadata.py "$project_name" "$repo_url" "$primary_language"
 
   # Run the main script with specific AI settings and experiment parameters
-  ./run.sh --ai-settings ai_settings.yaml -c -l 40 -m json_file --experiment-file "project_meta_data.json"
+  run_with_retries "./run.sh --ai-settings ai_settings.yaml -c -l \"$num\" -m json_file --experiment-file \"project_meta_data.json\"" "$project_name"
 
-elif [[ -f "$1" ]]; then
+elif [[ -f "$repo_url" ]]; then
   # Handle the case where the input is a file containing multiple repositories
-  file_path="$1"
+  file_path="$repo_url"
   echo "Using file path: $file_path"  # Print the file path being processed
 
   # Read the file line by line
@@ -66,7 +130,7 @@ elif [[ -f "$1" ]]; then
       python3.10 clone_and_set_metadata.py "$project_name" "$github_url" "$language"
 
       # Run the main script with specific AI settings and experiment parameters
-      ./run.sh --ai-settings ai_settings.yaml -c -l 40 -m json_file --experiment-file "project_meta_data.json"
+      run_with_retries "./run.sh --ai-settings ai_settings.yaml -c -l \"$num\" -m json_file --experiment-file \"project_meta_data.json\"" "$project_name"
   done < "$file_path"
 
 else

README.md

@@ -29,11 +29,11 @@ To get started in a VSCode Dev Container:
 ### 1️⃣ Single Repository Mode  
 You can directly process a single GitHub repository using the `--repo` option:  
 ```bash
-./ExecutionAgent.sh --repo <github_repo_url>
+./ExecutionAgent.sh --repo <github_repo_url> -l <num_value>
 ```  
 Example:  
 ```bash
-./ExecutionAgent.sh --repo https://github.com/pytest-dev/pytest
+./ExecutionAgent.sh --repo https://github.com/pytest-dev/pytest -l 50
 ```  
 
 When this mode is used, ExecutionAgent will:  
@@ -42,6 +42,8 @@ When this mode is used, ExecutionAgent will:
 3. Clone the repository and set up metadata.  
 4. Launch the main loop of ExecutionAgent to build the project and run its test cases.  
 
+The `-l` option allows you to specify the number of cycles which corresponds to the number of actions the agent can execute. By default, if `-l` is not provided, it will be set to 40. If you want to set a different number, simply pass the desired value after `-l`. For example, `-l 50` will use 50 instead of the default value.  
+
 ### 2️⃣ Batch File Mode  
 Prepare a batch file listing projects to process in the format:  
 `<project_name> <github_url> <language>`  
@@ -51,14 +53,13 @@ Example (notice how for now we leave one empty line after each entry):
 scipy https://github.com/scipy/scipy Python
 
 pytest https://github.com/pytest-dev/pytest Python
-
 ```
 
 Run ExecutionAgent with the batch file:  
 ```bash
 ./ExecutionAgent.sh /path/to/batch_file.txt
 ```  
-ExecutionAgent will process each project listed in the file, performing the same steps as the single repository mode.
+ExecutionAgent will process each project listed in the file, performing the same steps as the single repository mode. The `-l` option can also be applied here by adding it to the command when running the script.
 
 ---
 
@@ -77,6 +78,23 @@ ExecutionAgent will process each project listed in the file, performing the same
 
 Results are logged under `experimental_setups/experiment_XX`, where `XX` is an incremented number for each invocation of ExecutionAgent.  
 
+## 📁 Output Folder Structure Explanation  
+
+The folder structure under `experimental_setups/experiment_XX` is organized to keep track of the various outputs and logs generated during the execution of the `ExecutionAgent`. Below is a breakdown of the key directories and their contents:  
+
+- **files**: Contains files generated by the ExecutionAgent, such as `Dockerfile`, installation scripts, or any configuration files necessary for setting up the container environment.  
+  - Example: `Dockerfile`, `INSTALL.sh`   
+
+- **logs**: Stores raw logs capturing the input prompts and the corresponding outputs from the model during execution. These logs are essential for troubleshooting and understanding the behavior of the agent.  
+  - Example: `cycles_list_marshmallow`, `prompt_history_marshmallow`  
+
+- **responses**: Holds the responses generated by the model during the execution process in a structured JSON format. These responses include details about the generated build or test configurations and results.  
+  - Example: `model_responses_marshmallow`  
+
+- **saved_contexts**: Contains the saved states of the agent object at each iteration of the execution process. These snapshots are useful for debugging, tracking changes, and extracting subcomponents of the prompt across different cycles.  
+  - Example: `cycle_1`, `cycle_10`, etc.  
+
+This structure helps in managing the various data generated throughout the experimentation, ensuring easy access to logs, model responses, and agent states.
 ---
 
 ## 📜 Research Paper  

autogpt/app/main.py

@@ -295,6 +295,7 @@ def graceful_agent_interrupt(signum: int, frame: Optional[FrameType]) -> None:
         if cycles_remaining == 1:  # Last cycle
             stop_and_remove(agent.container)
             os.system("docker system prune -f")
+            exit()
             user_feedback, user_input, new_cycles_remaining = get_user_feedback(
                 config,
                 ai_config,

autogpt/commands/system.py

@@ -63,4 +63,6 @@ def task_complete(reason: str, agent: Agent) -> NoReturn:
     logger.info(title="Shutting down...\n", message=reason)
     stop_and_remove(agent.container)
     os.system("docker system prune -f")
+    with open(os.path.join("experimental_setups", agent.exp_number, "saved_contexts", "SUCCESS"), "w") as ssf:
+        ssf.write("SUCCESS")
     quit()

clean.sh

@@ -2,4 +2,5 @@ rm -rf experimental_setups/experiment_*
 rm -rf logs/*
 rm -rf execution_agent_workspace/*
 touch execution_agent_workspace/readme
-python3.10 remove_api_token.py
+python3.10 remove_api_token.py
+rm model_logging_temp.txt

post_process.py

@@ -0,0 +1,113 @@
+import os
+import json
+import sys
+import argparse
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import openai
+
+def ask_chatgpt(query, system_message, model="gpt-4"):
+    # Read the OpenAI API token from a file
+    with open("openai_token.txt") as opt:
+        token = opt.read().strip()
+
+    # Set up the OpenAI API key
+    openai.api_key = token
+
+    # Construct the messages for the Chat Completion API
+    messages = [
+        {"role": "system", "content": system_message},
+        {"role": "user", "content": query}
+    ]
+
+    # Call the OpenAI API for chat completion
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=messages
+    )
+
+    # Extract and return the content of the assistant's response
+    return response["choices"][0]["message"]["content"]
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: python post_process.py <project_name>")
+        sys.exit(1)
+
+    project_name = sys.argv[1]
+
+    # Read the last line of experiments_list.txt
+    experiments_file = "experimental_setups/experiments_list.txt"
+    if not os.path.exists(experiments_file):
+        print(f"Error: {experiments_file} does not exist.")
+        sys.exit(1)
+
+    with open(experiments_file, 'r') as f:
+        lines = f.readlines()
+        if not lines:
+            print(f"Error: {experiments_file} is empty.")
+            sys.exit(1)
+        last_line = lines[-1].strip()
+
+    # Build paths
+    success_file = f"experimental_setups/{last_line}/saved_contexts/{project_name}/SUCCESS"
+
+    if os.path.exists(success_file):
+        print("SUCCESS")
+        return
+
+    # Find the cycle_XX file with the highest XX
+    contexts_dir = f"experimental_setups/{last_line}/saved_contexts/{project_name}"
+    if not os.path.exists(contexts_dir):
+        print(f"Error: {contexts_dir} does not exist.")
+        sys.exit(1)
+
+    cycle_files = [f for f in os.listdir(contexts_dir) if f.startswith("cycle_") and f[6:].isdigit()]
+    if not cycle_files:
+        print(f"Error: No cycle files found in {contexts_dir}.")
+        sys.exit(1)
+
+    latest_cycle_file = max(cycle_files, key=lambda x: int(x[6:]))
+    latest_cycle_path = os.path.join(contexts_dir, latest_cycle_file)
+
+    # Read the JSON content of the latest cycle file
+    with open(latest_cycle_path, 'r') as f:
+        try:
+            file_content = json.load(f)
+        except json.JSONDecodeError:
+            print(f"Error: Failed to decode JSON from {latest_cycle_path}.")
+            sys.exit(1)
+
+    # Extract the desired content
+    try:
+        extracted_content = file_content["steps_object"]["1"]["result_of_step"]
+    except KeyError as e:
+        print(f"Error: Missing key in JSON structure: {e}")
+        sys.exit(1)
+
+    # Prepare the query for ask_chatgpt
+    query = (
+        "the following would represent the sequence of commands and reasoning made by an LLM trying to install \"webpack\" project from source code and execute test cases. "
+        "I want you to summarize the encountered problems and give advice for next attempt. Be precise and concise. Address the most important and critical issues (ignore non critical warnings and so). Your response should have one header: ### Feedback from previous installation attempts\n"
+        f"+ {extracted_content}"
+    )
+    
+    system_message = (
+        "You are a helpful software engineering assistant with capabilities of installing, building, configuring, and testing software projects."
+    )
+
+    # Call ask_chatgpt
+    response = ask_chatgpt(query, system_message)
+
+    # Save the response to problems_memory/{project_name}
+    problems_memory = f"problems_memory/{project_name}"
+    with open(problems_memory, 'w') as f:
+        f.write(response)
+
+    # Print FAILURE
+    print("FAILURE")
+
+if __name__ == "__main__":
+    main()

problems_memory/marshmallow

@@ -0,0 +1,11 @@
+### Feedback from previous installation attempts
+
+All provided reports suggest that the installation worked as expected; though, it's unclear if the "webpack" project is used at all since the contents of the reports all mention the 'marshmallow' project. 'tox' is used, which is a test runner, so it's assumed that the tests have been executed without stating any issues.
+
+If "webpack" should be installed and tested, the steps might have been unclear or misreported. "Webpack" is not a Python project, so Python-specific instructions or tools such as 'tox', 'venv', or 'pip' are not applicable. Webpack is a static module bundler for modern JavaScript applications and needs Node.js environment.
+
+Advice for next installation and test execution attempt:
+
+1. Check the scope of the project. Make sure you work on the correct software project ("webpack" vs. "marshmallow").
+2. If the objective is to install and test "webpack", Node.js and npm (node package manager) are required. Webpack can be installed globally with `npm install --global webpack`. For testing, use the included scripts in the project's `package.json` file, often it's as simple as running `npm test`.
+3. If working with the Python 'marshmallow' project, continue using Python environment setup with venv and pip. Use tox for executing the provided tests.

problems_memory/pytest

@@ -0,0 +1,13 @@
+### Feedback from previous installation attempts
+
+1. The initial setup and installation of the 'pytest' project seem to be successful with all the important dependencies correctly extracted. However, it is important to note that there is confusion between 'webpack' and 'pytest'. As per the request, the 'webpack' project was to be installed but all actions and results are related to 'pytest'.
+   
+2. Another issue is running pip as the root user, which is considered a bad practice. The warning suggests using a virtual environment, which helps in isolating the environment to prevent conflicts between different version of packages.
+
+3. In the later stages, the 'tox' command was not found which indicates that it may not have been installed or is not available in the current environment. This was corrected in a subsequent step by installing 'tox'.
+
+For the next attempt, ensure that:
+
+- The correct project ('webpack' vs 'pytest') is being installed.
+- Utilize a virtual environment to manage Python dependencies 
+- Verify that all necessary testing tools like 'tox' are installed before running test cases.

run.sh

@@ -25,7 +25,7 @@ if $PYTHON_CMD -c "import sys; sys.exit(sys.version_info < (3, 10))"; then
         $PYTHON_CMD -m pip install -r requirements.txt
     fi
     $PYTHON_CMD -m autogpt --skip-news "$@"
-    read -p "Press any key to continue..."
+    #read -p "Press any key to continue..."
 else
     echo "Python 3.10 or higher is required to run Auto GPT."
 fi