ADD: aws downloader, REF: big change, logging

.importlinter

@@ -173,3 +173,31 @@ forbidden_modules =
     adapt.visualization
     adapt.cli
     adapt.extensions
+
+# ==========================================================
+# 8. Downloaders are a self-contained third-party I/O leaf
+# ==========================================================
+#
+# adapt.downloaders isolates all boto3/S3 calls. It is a leaf:
+# it depends only on third-party libraries and stdlib, never on
+# any adapt internal, so it can be reused without dragging in
+# the rest of the package and never creates an import cycle.
+
+[importlinter:contract:downloaders_are_leaf]
+name = Downloaders package imports no adapt internals
+type = forbidden
+source_modules =
+    adapt.downloaders
+forbidden_modules =
+    adapt.modules
+    adapt.runtime
+    adapt.persistence
+    adapt.configuration
+    adapt.execution
+    adapt.api
+    adapt.gui
+    adapt.visualization
+    adapt.cli
+    adapt.extensions
+    adapt.contracts
+    adapt.utils

.pre-commit-config.yaml

@@ -18,6 +18,15 @@ repos:
 
   - repo: local
     hooks:
+      - id: forbid-adapt-allcaps
+        name: forbid all-caps product name (use "Adapt")
+        language: pygrep
+        entry: '\bADAPT\b'
+        types: [text]
+        # Source + docs only; the enforcement tests and this config must name the
+        # literal to forbid it, so they are intentionally out of scope.
+        files: ^(src/adapt/|docs/)
+
       - id: mypy
         name: mypy
         entry: mypy

docs/api/client.rst

@@ -1,7 +1,7 @@
 Repository Client API
 =====================
 
-Read-only interface for querying ADAPT pipeline output from a repository.
+Read-only interface for querying Adapt pipeline output from a repository.
 Initialise :class:`~adapt.api.RepositoryClient` with the repository root path;
 it auto-discovers runs, radars, scans, and data items through the two-tier
 database system (root-level registry + per-radar catalog).

environment.yml

@@ -28,9 +28,9 @@ dependencies:
   - contextily
   - pyarrow
   - duckdb
+  - boto3
   - pydata-sphinx-theme
   - pydantic
   - pip:
-      - nexradaws
       - sphinx-autodoc-typehints
       - myst-parser>=2.0

pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "pydantic>=2.0",
     "arm_pyart",
     "opencv-python",
-    "nexradaws",
+    "boto3",
     "pyarrow",
     "duckdb",
 ]
@@ -80,7 +80,7 @@ namespaces = false
 
 [tool.setuptools.package-data]
 "adapt.data" = ["user_config.py"]
-"adapt.schemas" = ["*.sql"]
+"adapt.configuration.schemas" = ["*.sql"]
 "adapt.config" = ["*.yaml"]
 
 [tool.setuptools_scm]

requirements.txt

@@ -7,7 +7,7 @@ scipy
 
 # NEXRAD & Radar
 arm_pyart
-nexradaws
+boto3   # anonymous S3 access for the native NEXRAD downloader
 
 # Image processing
 opencv-python

src/adapt/__init__.py

@@ -11,6 +11,15 @@
 Authors: Bhupendra Raut and Sid Gupta
 """
 
+import os as _os
+
+# Quiet third-party import-time chatter before any submodule (and its transitive
+# deps) load. Py-ART prints a citation banner on import unless PYART_QUIET is set,
+# and the ingest module imports pyart at its own import time — earlier than any
+# Adapt module could set this. The package root is the one place guaranteed to run
+# first. setdefault preserves a user-provided override.
+_os.environ.setdefault("PYART_QUIET", "1")
+
 import importlib.metadata as _importlib_metadata
 
 # Get the version

src/adapt/api/client.py

@@ -1,7 +1,7 @@
 # Copyright © 2026, UChicago Argonne, LLC
 # See LICENSE for terms and disclaimer.
 
-"""RepositoryClient — read-only access to an ADAPT repository.
+"""RepositoryClient — read-only access to an Adapt repository.
 
 Discovers data through the two-tier database system:
 - Root-level registry (adapt_registry.db): runs and radars.
@@ -53,15 +53,15 @@
 
 
 class RepositoryClient:
-    """Read-only interface for an ADAPT repository.
+    """Read-only interface for an Adapt repository.
 
     Thread-safe for notebook usage.
     Discovers all data through catalog databases — no filesystem inspection.
 
     Parameters
     ----------
     repository_root : str or Path
-        Root directory of the ADAPT repository.
+        Root directory of the Adapt repository.
     """
 
     def __init__(self, repository_root: str | Path) -> None:
@@ -494,7 +494,7 @@ def _bundle_from_scan_record(
             run_id=str(scan_record.get("run_id", "")),
             n_cells=int(scan_record.get("num_cells") or 0),
             max_reflectivity=float(scan_record.get("max_reflectivity") or 0.0),
-            has_tracks=bool(scan_record.get("has_tracks") or False),
+            has_tracks=bool(scan_record.get("has_tracks")),
         )
         seg = self._load_item_file(radar, scan_record.get("segmentation2d_item_id"))
         cells = self._load_item_file(radar, scan_record.get("analysis2d_item_id"))

src/adapt/api/domain.py

@@ -1,7 +1,7 @@
 # Copyright © 2026, UChicago Argonne, LLC
 # See LICENSE for terms and disclaimer.
 
-"""First-class domain objects for the ADAPT repository API."""
+"""First-class domain objects for the Adapt repository API."""
 
 from __future__ import annotations
 

src/adapt/cli.py

@@ -148,6 +148,14 @@ def _build_run_nexrad_parser(sub: argparse.ArgumentParser) -> None:
 
 def _run_nexrad(args: argparse.Namespace) -> None:
     """Execute the NEXRAD processing pipeline."""
+    from adapt import __version__
+    from adapt.runtime.run_reporter import format_banner
+
+    # Plain banner, first line of output — printed before logging is configured so it
+    # carries no log prefix and precedes any library/catalog chatter. Trailing blank
+    # line separates it cleanly from the run logs that follow.
+    print(format_banner(__version__) + "\n")
+
     if getattr(args, "only_modules", None) and getattr(args, "exclude_modules", None):
         raise SystemExit("error: --only and --not are mutually exclusive")
     _check_single_instance()
@@ -164,12 +172,6 @@ def _run_nexrad(args: argparse.Namespace) -> None:
 
     stop_event = threading.Event()
 
-    def _safe_stop(orch: PipelineOrchestrator) -> None:
-        try:
-            orch.stop()
-        except Exception as exc:
-            print(f"[adapt] Stop cleanup error (ignored): {exc}")
-
     def _run_orchestrator(
         orch: PipelineOrchestrator, max_runtime: int, done: threading.Event
     ) -> None:
@@ -182,7 +184,9 @@ def _handle_sigterm(signum, frame) -> None:
         print("\n[adapt] SIGTERM received — stopping pipeline...")
         orchestrator._interrupted = True
         stop_event.set()
-        threading.Thread(target=_safe_stop, args=(orchestrator,), daemon=True).start()
+        # Ask the orchestrator's own (joined) thread to stop; it runs finalize + the
+        # run summary in its start() finally, so the summary completes before exit.
+        orchestrator.request_stop()
 
     signal.signal(signal.SIGTERM, _handle_sigterm)
 
@@ -227,9 +231,10 @@ def _handle_sigterm(signum, frame) -> None:
         # Mark interrupted so the run is finalised as "cancelled" not "completed".
         orchestrator._interrupted = True
         stop_event.set()
-        # The orchestrator runs in a worker thread and never receives
-        # KeyboardInterrupt; set its stop flag explicitly.
-        threading.Thread(target=_safe_stop, args=(orchestrator,), daemon=True).start()
+        # The orchestrator runs in a worker thread and never receives KeyboardInterrupt.
+        # Ask it to break its loop; its own start() finally then runs stop() + the run
+        # summary on this (non-daemon, joined) thread — so the summary always prints.
+        orchestrator.request_stop()
         try:
             orchestrator_thread.join(timeout=20)
         except KeyboardInterrupt:
@@ -340,8 +345,6 @@ def _build_dashboard_parser(sub: argparse.ArgumentParser) -> None:
 
 def _dashboard_cmd(args: argparse.Namespace) -> None:
     """Launch the Adapt GUI dashboard."""
-    import os
-
     try:
         os.getcwd()
     except FileNotFoundError:

src/adapt/configuration/schemas/initialization.py

@@ -63,8 +63,6 @@ def write_default_config(path: Path, extensions: list[str] | None = None) -> Non
     own params under ``module_params``. Public — called by both
     ``init_runtime_config`` (auto-bootstrap) and ``adapt config``.
     """
-    from datetime import datetime as _dt
-
     from adapt.configuration.schemas import yaml_writer
     from adapt.configuration.schemas.assemble import (
         assemble_default_config,
@@ -76,7 +74,7 @@ def write_default_config(path: Path, extensions: list[str] | None = None) -> Non
     # config.yaml` works without --base-dir; the user can edit or override it.
     data = {"base_dir": str(path.parent.resolve()), **data}
     descriptions = assemble_descriptions(extensions)
-    header = _CONFIG_HEADER.format(timestamp=_dt.now().strftime("%Y-%m-%d %H:%M:%S"))
+    header = _CONFIG_HEADER.format(timestamp=datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(yaml_writer.dump(data, descriptions, header=header))
@@ -99,7 +97,7 @@ def _load_user_config_dict(config_path: str) -> dict:
             raise ImportError(
                 "PyYAML is required for YAML config files: pip install pyyaml"
             ) from err
-        with open(path) as f:
+        with open(path, encoding="utf-8") as f:
             data = yaml.safe_load(f)
         return data or {}
 
@@ -187,7 +185,7 @@ def _persist_runtime_config(
     config_dict["run_id"] = run_id
     config_dict["created_at"] = datetime.now(UTC).isoformat()
 
-    with open(config_file, "w") as f:
+    with open(config_file, "w", encoding="utf-8") as f:
         json.dump(config_dict, f, indent=2, default=str)
 
 
@@ -215,7 +213,7 @@ def _find_matching_run_id(new_config_dict: dict) -> str | None:
 
     for cfg_file in candidates:
         try:
-            with open(cfg_file) as f:
+            with open(cfg_file, encoding="utf-8") as f:
                 saved = json.load(f)
             if _config_fingerprint(saved) == target:
                 return saved.get("run_id")
@@ -238,7 +236,7 @@ def _load_saved_runtime_config(base_dir: str, run_id: str) -> InternalConfig:
     if not cfg_path.exists():
         raise FileNotFoundError(f"Saved runtime config not found for run_id '{run_id}': {cfg_path}")
 
-    with open(cfg_path) as f:
+    with open(cfg_path, encoding="utf-8") as f:
         cfg_dict = json.load(f)
 
     # Non-schema metadata persisted for audit only.

src/adapt/configuration/schemas/internal.py

@@ -36,6 +36,7 @@ class InternalDownloaderConfig(AdaptBaseModel):
     latest_files: int
     latest_minutes: int
     poll_interval_sec: int
+    max_fetch_retries: int
     start_time: str | None
     end_time: str | None
     min_file_size: int
@@ -50,6 +51,7 @@ class InternalRegridderConfig(AdaptBaseModel):
     min_radius: float
     weighting_function: Literal["cressman", "barnes", "nearest"]
     save_netcdf: bool
+    netcdf_save_retries: int
 
 
 class InternalSegmenterConfig(AdaptBaseModel):
@@ -140,6 +142,13 @@ class InternalCellUidConfig(AdaptBaseModel):
     core_reflectivity_threshold: float = Field(default=40.0, ge=0.0)
     max_gap_minutes: float = Field(default=10.0, gt=0.0)
     expected_speed_ms: float = Field(default=30.0, gt=0.0)
+    max_tracking_gap_minutes: float = Field(default=20.0, gt=0.0)
+    projection_horizon_minutes: float = Field(default=20.0, gt=0.0)
+    projection_interval_minutes: float = Field(default=1.0, gt=0.0)
+    max_speed_ms: float = Field(default=40.0, gt=0.0)
+    max_speed_multiplier: float = Field(default=3.0, gt=0.0)
+    overlap_match_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
+    heading_change_penalty_weight: float = Field(default=0.0, ge=0.0)
     cell_uid: InternalCellUidConfig
 
 
@@ -169,9 +178,22 @@ class InternalOutputConfig(AdaptBaseModel):
 
 
 class InternalLoggingConfig(AdaptBaseModel):
-    """Runtime logging configuration."""
+    """Runtime logging + observability configuration.
+
+    ``level`` governs the full file/JSON log; ``console_level`` keeps the console
+    quiet independently. The remaining toggles enable/disable the observability
+    subsystem and its pillars; the orchestrator translates these into an
+    ``ObsSettings`` when it builds the provider.
+    """
 
     level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    console_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "WARNING"
+    enabled: bool = True
+    traces: bool = True
+    metrics: bool = True
+    json_logs: bool = False
+    console_logs: bool = True
+    progress_every: float = Field(default=30.0, gt=0.0)
 
 
 class InternalProcessorConfig(AdaptBaseModel):

src/adapt/configuration/schemas/param.py

@@ -35,6 +35,9 @@ class DownloaderConfig(AdaptBaseModel):
     latest_files: int = Field(5, ge=1, description="Number of latest files to keep")
     latest_minutes: int = Field(60, ge=1, description="Time window in minutes")
     poll_interval_sec: int = Field(300, ge=1, description="Polling interval in seconds")
+    max_fetch_retries: int = Field(
+        3, ge=1, description="AWS scan-fetch attempts before giving up for this poll"
+    )
     start_time: str | None = None
     end_time: str | None = None
     min_file_size: int = Field(
@@ -55,6 +58,9 @@ class RegridderConfig(AdaptBaseModel):
     min_radius: float = Field(1750.0, gt=0)
     weighting_function: Literal["cressman", "barnes", "nearest"] = "cressman"
     save_netcdf: bool = True
+    netcdf_save_retries: int = Field(
+        3, ge=1, description="NetCDF write attempts before raising (when save_netcdf is set)"
+    )
 
 
 class SegmenterConfig(AdaptBaseModel):
@@ -227,6 +233,45 @@ class CellUidConfig(AdaptBaseModel):
         gt=0.0,
         description="Maximum expected cell propagation speed (m/s); scales D_pos with dt",
     )
+    max_tracking_gap_minutes: float = Field(
+        20.0,
+        gt=0.0,
+        description="Hard limit: scan gaps above this terminate all tracks and restart "
+        "(no matching attempted across the gap)",
+    )
+    projection_horizon_minutes: float = Field(
+        20.0,
+        gt=0.0,
+        description="How far ahead (minutes) registration-based projected hulls are consumed",
+    )
+    projection_interval_minutes: float = Field(
+        1.0,
+        gt=0.0,
+        description="Spacing (minutes) between registration projected hulls",
+    )
+    max_speed_ms: float = Field(
+        40.0,
+        gt=0.0,
+        description="Hard physical cap (m/s); candidate pairs above this are rejected pre-matching",
+    )
+    max_speed_multiplier: float = Field(
+        3.0,
+        gt=0.0,
+        description="Hard acceleration cap: reject if candidate speed exceeds this times the "
+        "track's previous speed",
+    )
+    overlap_match_threshold: float = Field(
+        0.3,
+        ge=0.0,
+        le=1.0,
+        description="Min projected-hull overlap for a deterministic unique-overlap direct match "
+        "(uniqueness dominates the threshold)",
+    )
+    heading_change_penalty_weight: float = Field(
+        0.0,
+        ge=0.0,
+        description="Optional cost penalty per radian of heading change (0 = diagnostic only)",
+    )
     cell_uid: CellUidConfig = Field(default_factory=CellUidConfig)  # type: ignore[arg-type]