patches

.github/workflows/tests.yaml

@@ -0,0 +1,154 @@
+
+name: Testing
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  setup-build:
+    name: Ex1 (${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Cache MTH5 test files
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/aurora
+          key: mth5-test-files-${{ runner.os }}-${{ hashFiles('tests/conftest.py') }}
+          restore-keys: |
+            mth5-test-files-${{ runner.os }}-
+
+      - name: Create virtual environment and install dependencies
+        run: |
+          uv venv --python ${{ matrix.python-version }}
+          source .venv/bin/activate
+          uv pip install --upgrade pip
+          uv pip install -e ".[dev,test]"
+          # uv pip install mt_metadata[obspy]
+          uv pip install "mt_metadata[obspy] @ git+https://github.com/kujaku11/mt_metadata.git@patches"
+          uv pip install git+https://github.com/kujaku11/mth5.git@patches
+
+          # uv pip install mth5
+          uv pip install git+https://github.com/kujaku11/mth5_test_data.git
+          # Explicitly include nbconvert & ipykernel
+          uv pip install jupyter nbconvert nbformat ipykernel pytest pytest-cov pytest-timeout codecov
+          python -m ipykernel install --user --name "python3"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc
+
+      - name: Set kernel and execute Jupyter Notebooks
+        run: |
+          source .venv/bin/activate
+          python << 'EOF'
+          import nbformat
+          import subprocess
+          import sys
+
+          notebooks = [
+              "docs/examples/dataset_definition.ipynb",
+              "docs/examples/operate_aurora.ipynb",
+              "docs/tutorials/pkd_units_check.ipynb",
+              "docs/tutorials/pole_zero_fitting/lemi_pole_zero_fitting_example.ipynb",
+              "docs/tutorials/processing_configuration.ipynb",
+              "docs/tutorials/process_cas04_multiple_station.ipynb",
+              "docs/tutorials/synthetic_data_processing.ipynb"
+          ]
+
+          failures = []
+
+          for nb_path in notebooks:
+              # Update kernel spec
+              print(f"Updating kernel in {nb_path}")
+              try:
+                  with open(nb_path, "r", encoding="utf-8") as f:
+                      nb = nbformat.read(f, as_version=4)
+
+                  nb["metadata"]["kernelspec"]["name"] = "python3"
+                  nb["metadata"]["kernelspec"]["display_name"] = "Python (python3)"
+
+                  with open(nb_path, "w", encoding="utf-8") as f:
+                      nbformat.write(nb, f)
+                  print(f"✓ Updated kernel in {nb_path}")
+              except Exception as e:
+                  print(f"✗ Failed to update kernel in {nb_path}:  {e}")
+                  failures.append(nb_path)
+                  continue
+
+              # Execute notebook
+              print(f"Executing {nb_path}")
+              result = subprocess.run(
+                  ["jupyter", "nbconvert", "--to", "notebook", "--execute", nb_path],
+                  capture_output=True,
+                  text=True
+              )
+
+              if result.returncode != 0:
+                  print(f"✗ Failed to execute {nb_path}")
+                  print(result.stderr)
+                  failures.append(nb_path)
+              else:
+                  print(f"✓ Successfully executed {nb_path}")
+
+          if failures:
+              print("\n======= Summary =======")
+              print(f"Failed notebooks: {failures}")
+              sys.exit(1)
+          else:
+              print("\n✓ All notebooks executed successfully!")
+          EOF
+
+      - name: Run Tests
+        run: |
+          source .venv/bin/activate
+          pytest -s -v --cov=./ --cov-report=xml --cov=aurora -n auto tests
+
+      - name: "Upload coverage reports to Codecov"
+        uses: codecov/codecov-action@v4
+        with:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
+          flags: tests
+
+      # Note: these conditions won't match python-version 3.10; adjust if desired.
+      - name: Build Doc
+        if: ${{ (github.ref == 'refs/heads/main') && (matrix.python-version == '3.8') }}
+        run: |
+          source .venv/bin/activate
+          cd docs
+          make html
+          cd ..
+
+      - name: GitHub Pages
+        if: ${{ (github.ref == 'refs/heads/main') && (matrix.python-version == '3.8') }}
+        uses: crazy-max/ghaction-github-pages@v2.5.0
+        with:
+          build_dir: docs/_build/html
+          jekyll: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -1,10 +1,45 @@
+# .pre-commit-config.yaml
 repos:
--   repo: https://github.com/ambv/black
-    rev: 22.6.0
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
     hooks:
-    - id: black
-      language_version: python3.10
--   repo: https://github.com/pycqa/flake8
-    rev: 3.9.2
+    -   id: trailing-whitespace
+        types: [python]
+    -   id: end-of-file-fixer
+        types: [python]
+    -   id: check-yaml
+        exclude: '^(?!.*\.py$).*$'
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-    - id: flake8
+    -   id: isort
+        types: [python]
+        exclude: (__init__.py)$
+        files: \.py$
+        args: ["--profile", "black",
+               "--skip-glob","*/__init__.py",
+               "--force-alphabetical-sort-within-sections",
+               "--order-by-type",
+               "--lines-after-imports=2"]
+
+-   repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+    -   id: black
+        types: [python]
+        files: \.py$
+        language_version: python3
+
+-   repo: https://github.com/pycqa/autoflake
+    rev: v2.1.1
+    hooks:
+    -   id: autoflake
+        types: [python]
+        files: \.py$
+        args: [
+            "--remove-all-unused-imports",
+            "--expand-star-imports",
+            "--ignore-init-module-imports",
+            "--in-place"
+        ]

aurora/__init__.py

@@ -13,7 +13,7 @@
             "sink": sys.stdout,
             "level": "INFO",
             "colorize": True,
-            "format": "<level>{time} | {level: <3} | {name} | {function} | {message}</level>",
+            "format": "<level>{time} | {level: <3} | {name} | {function} | line: {line} | {message}</level>",
         },
     ],
     "extra": {"user": "someone"},

aurora/config/config_creator.py

@@ -16,7 +16,7 @@
 from aurora.config.metadata import Processing
 from aurora.sandbox.io_helpers.emtf_band_setup import EMTFBandSetupFile
 from mth5.processing.kernel_dataset import KernelDataset
-from mt_metadata.transfer_functions.processing.window import Window
+from mt_metadata.processing.window import Window
 
 import pathlib
 
@@ -127,11 +127,13 @@ def create_from_kernel_dataset(
         kernel_dataset: KernelDataset,
         input_channels: Optional[list] = None,
         output_channels: Optional[list] = None,
+        remote_channels: Optional[list] = None,
         estimator: Optional[str] = None,
         emtf_band_file: Optional[Union[str, pathlib.Path]] = None,
         band_edges: Optional[dict] = None,
         decimation_factors: Optional[list] = None,
         num_samples_window: Optional[int] = None,
+        **kwargs,
     ) -> Processing:
         """
         This creates a processing config from a kernel dataset.
@@ -166,6 +168,8 @@ def create_from_kernel_dataset(
             List of the input channels that will be used in TF estimation (usually "hx", "hy")
         output_channels: list
             List of the output channels that will be estimated by TF (usually "ex", "ey", "hz")
+        remote_channels: list
+            List of the remote reference channels (usually "hx", "hy" at remote site)
         estimator:  Optional[Union[str, None]]
             The name of the regression estimator to use for TF estimation.
         emtf_band_file: Optional[Union[str, pathlib.Path, None]]
@@ -176,6 +180,12 @@ def create_from_kernel_dataset(
             List of decimation factors, normally [1, 4, 4, 4, ... 4]
         num_samples_window: Optional[Union[int, None]]
             The size of the window (usually for FFT)
+        **kwargs:
+            Additional keyword arguments passed to Processing constructor. Could contain:
+            - save_fcs: bool
+               - If True, save Fourier coefficients during processing.
+            - save_fcs_type: str
+                - File type for saving Fourier coefficients.  Options are "h5" or "csv".
 
         Returns
         -------
@@ -241,8 +251,17 @@ def create_from_kernel_dataset(
             else:
                 decimation_obj.output_channels = output_channels
 
+            if remote_channels is None:
+                if kernel_dataset.remote_channels is not None:
+                    decimation_obj.reference_channels = kernel_dataset.remote_channels
+
             if num_samples_window is not None:
                 decimation_obj.stft.window.num_samples = num_samples_window[key]
+
+            if kwargs.get("save_fcs", False):
+                decimation_obj.save_fcs = True
+                decimation_obj.save_fcs_type = kwargs.get("save_fcs_type", "h5")
+
             # set estimator if provided as kwarg
             if estimator:
                 try: