Pytest latest

.github/workflows/docker_build.yaml

@@ -0,0 +1,28 @@
+name: Build Docker Image
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Build docker image
+      run: docker build -f Dockerfile.minimal -t acr .
+
+    - name: Start docker image (background)
+      run: docker run --name acr-pytest -t -d acr 
+
+    # Due to diffuculties with `conda activate` in docker, we do `conda run` while specifying the environment
+    # setting cov-report to term and xml -> outputs coverage report to terminal, and an xml file inside the container
+    - name: Run PyTest with Coverage (inside docker)
+      run: docker exec acr-pytest conda run --no-capture-output -n auto-code-rover pytest --cov=app test/ --cov-report=term --cov-report=xml

.github/workflows/pytest.yml

@@ -4,14 +4,19 @@ on:
   push:
     branches:
       - main
-      - pytest-ci
+      - pytest-latest
+      - pytest-integration
   pull_request:
     branches:
       - main
 
 jobs:
   pytest:
     runs-on: ubuntu-latest
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
+      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
     defaults:
       run:
         shell: bash -l {0}
@@ -41,7 +46,18 @@ jobs:
         run: conda install -y tox
 
       - name: Run tox tests
-        run: tox -e py
+        run: |
+          echo "Current branch: ${{ github.ref }}"
+          if [[ "${{ github.ref }}" == "refs/heads/pytest-latest" ]]; then
+            echo "Running unit tests only."
+            tox -e unit
+          elif [[ "${{ github.ref }}" == "refs/heads/pytest-integration" ]]; then
+            echo "Running integration tests only."
+            tox -e integration
+          else
+            echo "Running both unit and integration tests."
+            tox -e unit,integration
+          fi
 
       - name: Check Coverage Report Exists
         run: |
@@ -50,6 +66,13 @@ jobs:
             exit 1
           fi
 
+      - name: Print Branch Coverage
+        run: |
+          echo "Branch Coverage Details:"
+          echo "Total Branches: $(grep -o 'branches-valid="[0-9]*"' coverage.xml | sed 's/branches-valid="//;s/"//')"
+          echo "Branches Covered: $(grep -o 'branches-covered="[0-9]*"' coverage.xml | sed 's/branches-covered="//;s/"//')"
+          echo "Branch Rate: $(grep -o 'branch-rate="[0-9\.]*"' coverage.xml | sed 's/branch-rate="//;s/"//')"
+
       - name: SonarQube Scan
         uses: SonarSource/sonarqube-scan-action@v4
         env:

.gitignore

@@ -159,4 +159,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-output/
+output/
+setup/

ACR.py

@@ -1,3 +1,4 @@
 from app import main
+
 if __name__ == "__main__":
     main.main()

README.md

@@ -338,6 +338,10 @@ The current list of supported models:
 
 Please refer to [EXPERIMENT.md](EXPERIMENT.md) for information on experiment replication.
 
+## Testing
+
+Please refer to [TESTING.md](TESTING.md) for information on setting up and running tests.
+
 ## ✉️ Contacts
 
 For any queries, you are welcome to open an issue.

TESTING.md

@@ -11,16 +11,43 @@ The testing suite uses the following libraries and tools:
 - Pytest, to execute the tests
 - Coverage, (the Coverage.py tool) to measure the code coverage
 
-In the `auto-code-rover` environment, install the required libraries by running the following command:
 
+Creating the `auto-code-rover` environment using the `environment.yaml` file:
 ```bash
+conda env create -f environment.yml
+conda activate auto-code-rover
+```
+
+
+In the `auto-code-rover` environment, add `conda-forge` as a channel, then install the required libraries by running the following command:
+
+```bash
+conda config --add channels conda-forge
+conda config --set channel_priority flexible
 conda install -y tox
 ```
 
-and execute the tox commands (configured in `tox.ini`) to run the tests:
+## Running the tests locally
+
+To run the tests, execute the `tox` command (you can view the configurations in `tox.ini`) to run the tests:
+
+```bash
+tox
+```
+
+Alternatively, command to activate the `auto-code-rover` environment and run the tests:
 
 ```bash
-tox -e py
+conda activate auto-code-rover && tox
 ```
 
-The test results and the test coverage report will be displayed in the terminal, with a `coverage.xml` file in the Cobertura format generated in the project's root directory.
+The test results and the test coverage report will be displayed in the terminal, with a `coverage.xml` file in the Cobertura format generated in the project's root directory.
+
+## Modifying the `tox.ini` file
+
+To enable missing statement coverage, add the following section the `tox.ini` file:
+
+```ini
+[coverage:report]
+show_missing = True
+```

app/data_structures.py

@@ -51,7 +51,7 @@ def __init__(
         )
 
     def __str__(self):
-        return f"Call function `{self.func_name}` with arguments {self.arg_values}."
+        return f"FunctionCallIntent(func_name={self.func_name}, arguments={self.arg_values})"
 
     def to_dict(self):
         return {"func_name": self.func_name, "arguments": self.arg_values}

app/search/search_utils.py

@@ -340,11 +340,11 @@ def get_code_region_around_line(
 
     # start and end should also be 1-based valid line numbers
     start = max(1, line_no - window_size)
-    end = min(len(file_content), line_no + window_size)
+    end = min(len(file_content) + 1, line_no + window_size)
     snippet = ""
     for i in range(start, end):
         if with_lineno:
             snippet += f"{i} {file_content[i - 1]}"
         else:
-            snippet += file_content[i]
+            snippet += file_content[i - 1]
     return snippet

environment.osx-arm64.yml

@@ -0,0 +1,22 @@
+name: auto-code-rover
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - bzip2=1.0.8
+  - ca-certificates=2024.7.2
+  - libffi=3.4.4
+  - ncurses=6.5
+  - openssl=3.3.1
+  - pip=24.2
+  - python=3.10.11
+  - readline=8.2
+  - setuptools=68.2.2
+  - sqlite=3.46.0
+  - tk=8.6.14
+  - tzdata=2023d
+  - wheel=0.41.2
+  - xz=5.4.6
+  - rich=13.8.1
+  - pip:
+      - -r requirements.osx-arm64.txt

pyrightconfig.json

@@ -0,0 +1,6 @@
+{
+    "include": ["app"],
+    "exclude": ["venv"],
+    "pythonVersion": "3.10",
+    "typeCheckingMode": "basic"
+  }

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+norecursedirs = setup output
+markers =
+    integration: mark a test as an integration test.

requirements.osx-arm64.txt

@@ -0,0 +1,115 @@
+aiohttp==3.10.2
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-tools==0.2.1
+anyio==4.2.0
+astroid==3.2.3
+attrs==23.2.0
+beautifulsoup4==4.12.3
+black==25.1.0
+Brotli
+certifi==2024.7.4
+cffi
+cfgv==3.4.0
+charset-normalizer
+click==8.1.7
+coverage==7.5.3
+Cython==3.0.8
+dill==0.3.8
+discord==2.3.2
+discord.py==2.3.2
+distlib==0.3.8
+distro==1.9.0
+docker==7.0.0
+docstring-parser==0.15
+emojis==0.7.0
+filelock==3.13.1
+frozenlist==1.4.1
+fsspec==2024.6.1
+h11==0.14.0
+h2
+hpack==4.0.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hyperframe
+icecream==2.1.3
+identify==2.5.33
+idna==3.7
+importlib-metadata==7.0.1
+install-jdk==1.1.0
+isort==5.13.2
+javalang==0.13.0
+Jinja2==3.1.4
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+libclang==18.1.1
+linkify-it-py==2.0.2
+litellm==1.44.8
+loguru==0.7.2
+lxml==5.1.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+more-itertools==10.2.0
+mpmath==1.3.0
+multidict==6.0.4
+networkx==3.2.1
+nodeenv==1.8.0
+numpy==1.26.4
+natsort==8.4.0
+ollama==0.3.3
+openai==1.50.2
+opt-einsum==3.3.0
+packaging==23.2
+platformdirs==4.1.0
+polars==0.20.31
+pre-commit==3.6.0
+pycparser
+pydantic==2.5.3
+pydantic_core==2.14.6
+Pygments==2.17.2
+pylint==3.2.3
+pyro-api==0.1.2
+pyro-ppl==1.9.0
+PySocks
+pyright==1.1.397
+pytest==8.3.4
+pytest-cov==6.0.0
+python-dotenv==1.0.0
+PyYAML==6.0.1
+referencing==0.32.1
+regex==2024.5.15
+rich==13.7.1
+rpds-py==0.16.2
+semver==3.0.2
+slack_sdk==3.26.2
+sniffio==1.3.0
+soupsieve==2.5
+sympy==1.13.0
+tenacity==8.2.3
+termcolor==2.4.0
+textual==0.52.1
+tiktoken==0.7.0
+timeout-decorator==0.5.0
+tokenizers==0.19.1
+tomlkit==0.13.0
+torch==2.2.1
+tqdm==4.66.4
+tree-sitter==0.21.3
+tree-sitter-c==0.21.4
+tree-sitter-cpp==0.22.2
+tree-sitter-java==0.21.0
+tree-sitter-languages==1.10.2
+types-jsonschema==4.21.0.20240311
+typing_extensions==4.12.2
+uc-micro-py==1.0.2
+unidiff==0.7.5
+unittest-xml-reporting==3.2.0
+urllib3
+virtualenv==20.25.0
+yarl==1.9.4
+zipp==3.19.2
+zstandard==0.22.0

test/app/agents/test_agent_common.py

@@ -1,24 +1,29 @@
-import pytest
 from app.data_structures import MessageThread
 from app.agents.agent_common import replace_system_prompt, InvalidLLMResponse
 
+
 def test_replace_system_prompt():
     # Setup: create a MessageThread with a system message and another message
     original_prompt = "Original System Prompt"
     new_prompt = "New System Prompt"
     messages = [
         {"role": "system", "content": original_prompt},
-        {"role": "user", "content": "Hello"}
+        {"role": "user", "content": "Hello"},
     ]
     msg_thread = MessageThread(messages=messages)
 
     # Execute: replace the system prompt
     updated_thread = replace_system_prompt(msg_thread, new_prompt)
 
     # Verify: first message should now have the new prompt
-    assert updated_thread.messages[0]["content"] == new_prompt, "System prompt was not replaced correctly."
+    assert (
+        updated_thread.messages[0]["content"] == new_prompt
+    ), "System prompt was not replaced correctly."
     # Verify: the rest of the messages remain unchanged
-    assert updated_thread.messages[1]["content"] == "Hello", "User message was unexpectedly modified."
+    assert (
+        updated_thread.messages[1]["content"] == "Hello"
+    ), "User message was unexpectedly modified."
+
 
 def test_replace_system_prompt_returns_same_object():
     # Setup: create a MessageThread with a single system message
@@ -30,4 +35,6 @@ def test_replace_system_prompt_returns_same_object():
     result = replace_system_prompt(msg_thread, new_prompt)
 
     # Verify: the same MessageThread instance is returned (in-place modification)
-    assert result is msg_thread, "replace_system_prompt should return the same MessageThread object."
+    assert (
+        result is msg_thread
+    ), "replace_system_prompt should return the same MessageThread object."