python-hatch-workflow.yml

# This workflow installs Python dependencies and runs tests across versions.
#
# Cost design (to stay within GitHub Actions included minutes):
#   - Pull requests:  fast tests only, single Python version  (cheap, quick gate)
#   - Push to main:   fast (full matrix) + network + slow      (real integration gate)
#   - Nightly (cron): full suite                               (freshness vs SEC API)
#   - concurrency:    cancel superseded runs on the same ref
#   - paths-ignore:   skip docs/markdown-only changes
#   - pip cache:      avoid reinstalling deps from scratch in every job

name: Build and Test Edgartools

on:
  push:
    branches: [ "main" ]
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - 'mkdocs.yml'
      - 'LICENSE'
      - '.gitignore'
  pull_request:
    branches: [ "main" ]
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - 'mkdocs.yml'
      - 'LICENSE'
      - '.gitignore'
  schedule:
    # Nightly full suite at 07:00 UTC
    - cron: '0 7 * * *'
  workflow_dispatch:

# Cancel an in-progress run when a newer commit lands on the same branch/PR.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test-fast:
    # PRs: single version for a quick gate. Push/schedule: full matrix.
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.13"]') || fromJSON('["3.10", "3.13"]') }}

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy<8.2" freezegun==1.5.1 filelock tqdm responses

    - name: Run fast tests
      run: |
        # Parallelize fast tests - safe, no SEC API calls
        pytest -n auto --cov --cov-report=xml -m 'fast and not regression'

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      if: matrix.python-version == '3.13' && github.event_name != 'pull_request'
      with:
        name: coverage-fast
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  test-network:
    # Network tests skip PRs — SEC API behavior doesn't change per-commit.
    # Runs on push to main, nightly schedule, and manual dispatch.
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    env:
      # Relocate the EDGAR data/cache dir into the workspace so it can be cached.
      # This only moves where the HTTP cache (_tcache) lives; it does NOT enable
      # local storage (that needs EDGAR_USE_LOCAL_DATA, which tests disable).
      EDGAR_LOCAL_DATA_DIR: ${{ github.workspace }}/.edgar-data

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python 3.12
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.12"
        cache: 'pip'

    # Persist the SEC HTTP cache across runs. Filing documents and XBRL under
    # /Archives/edgar/data are cached forever (see CACHE_RULES), so a warm cache
    # serves the bulk of the suite from disk instead of re-downloading from SEC.
    # run_id-suffixed key always writes a fresh entry; restore-keys warm-starts
    # from the most recent prior cache (monotonic, GH evicts LRU at the 10GB cap).
    - name: Restore SEC HTTP cache
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ github.workspace }}/.edgar-data/_tcache
        key: edgar-tcache-network-${{ github.run_id }}
        restore-keys: |
          edgar-tcache-network-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy<8.2" freezegun==1.5.1 filelock tqdm responses

    - name: Run network tests
      run: |
        # Sequential - respects SEC rate limits. --enable-cache persists HTTP
        # responses to _tcache (immutable /Archives data cached forever).
        pytest --cov --cov-report=xml -m 'network and not slow and not regression' --enable-cache

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: coverage-network
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  test-slow:
    # Slow tests skip PRs (often network-heavy). Push/schedule/dispatch only.
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    env:
      EDGAR_LOCAL_DATA_DIR: ${{ github.workspace }}/.edgar-data

    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python 3.12
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.12"
        cache: 'pip'

    # Persist the SEC HTTP cache (see test-network for rationale).
    - name: Restore SEC HTTP cache
      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
      with:
        path: ${{ github.workspace }}/.edgar-data/_tcache
        key: edgar-tcache-slow-${{ github.run_id }}
        restore-keys: |
          edgar-tcache-slow-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e ".[ai]"
        python -m pip install pytest pytest-cov pytest-env pytest-xdist pytest-asyncio pytest-retry pytest-mock pytest-vcr "vcrpy<8.2" freezegun==1.5.1 filelock tqdm responses

    - name: Run slow tests
      run: |
        # Sequential - often network-heavy. --enable-cache persists HTTP responses.
        pytest --cov --cov-report=xml -m 'slow and not regression' --enable-cache

    - name: Upload coverage data
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: coverage-slow
        path: |
          .coverage
          coverage.xml
        include-hidden-files: true
        retention-days: 1

  coverage:
    name: Combine Coverage
    # Only meaningful when the full suite ran (not on PRs).
    if: github.event_name != 'pull_request'
    needs: [test-fast, test-network, test-slow]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
    - name: Set up Python
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "3.11"
        cache: 'pip'
    - name: Install coverage
      run: pip install coverage[toml]

    - name: Download coverage data (fast)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-fast
        path: coverage-fast/

    - name: Download coverage data (network)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-network
        path: coverage-network/

    - name: Download coverage data (slow)
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: coverage-slow
        path: coverage-slow/

    - name: Combine coverage
      run: |
        # Move .coverage files with unique names
        mv coverage-fast/.coverage .coverage.fast
        mv coverage-network/.coverage .coverage.network
        mv coverage-slow/.coverage .coverage.slow

        # Combine all coverage data
        coverage combine

        # Generate report and check threshold
        coverage report --fail-under=65
        coverage xml -o combined-coverage.xml

    - name: Upload combined coverage to Codecov
      uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
      with:
        files: combined-coverage.xml
        fail_ci_if_error: false
        token: ${{ secrets.CODECOV_TOKEN }}