From 753fe0434a7c0f35b9256b676c04e80c9a62bd13 Mon Sep 17 00:00:00 2001 From: robcohen Date: Wed, 17 Dec 2025 10:34:17 -0800 Subject: [PATCH 1/2] Add fetch command for per-account JSON export Adds a new `simplefin fetch` command that: - Fetches all accounts with their transactions - Outputs one JSON file per account - Organizes files hierarchically: //_.json This structure is useful for integration with tools like beangulp that expect one file per account. The hierarchical layout is human-navigable and the timestamped filenames preserve history. Usage: simplefin fetch --output-dir /path/to/output --lookback-days 30 --- src/simplefin/cli/__init__.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/simplefin/cli/__init__.py b/src/simplefin/cli/__init__.py index c40d44a..ddfa896 100644 --- a/src/simplefin/cli/__init__.py +++ b/src/simplefin/cli/__init__.py @@ -127,5 +127,87 @@ def info() -> None: pprint(info) +@cli.command() +@click.option( + "--output-dir", + type=click.Path(file_okay=False, dir_okay=True), + required=True, + help="Directory to output per-account JSON files", +) +@click.option( + "lookback_days", + "--lookback-days", + type=int, + default=30, + help="Number of days to look back for transactions (default: 30)", +) +def fetch(output_dir: str, lookback_days: int) -> None: + """Fetch all accounts with transactions to separate JSON files. + + Creates one JSON file per account in the output directory, organized by + institution and account name: + + ///_.json + + This structure is human-navigable and preserves history. It's useful for + integration with tools like beangulp that expect one file per account. + """ + import pathlib + + def sanitize_path(name: str) -> str: + """Sanitize a string for use as a directory/file name.""" + safe = "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in name) + # Collapse multiple dashes and strip leading/trailing dashes + while "--" in safe: + safe = safe.replace("--", "-") + return safe.strip("-") + + c = SimpleFINClient(access_url=os.getenv("SIMPLEFIN_ACCESS_URL")) + console = Console() + + # Get list of accounts (with balance info but no transactions) + accounts = c.get_accounts() + console.print(f"Found {len(accounts)} accounts") + + # Create output directory + output_path = pathlib.Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # Fetch transactions for each account + start_dt = datetime.date.today() - datetime.timedelta(days=lookback_days) + today_str = datetime.date.today().isoformat() + + for account in accounts: + account_id = account["id"] + account_name = account["name"] + org_domain = account.get("org", {}).get("domain", "unknown") + + # Fetch transactions for this account + # get_transactions returns a list of transaction dicts + transactions = c.get_transactions(account_id, start_dt) + + # Merge account metadata with transactions + account_data = account.copy() + account_data["transactions"] = transactions if isinstance(transactions, list) else [] + + # Build directory structure: // + inst_dir = output_path / sanitize_path(org_domain) + acct_dir = inst_dir / sanitize_path(account_name) + acct_dir.mkdir(parents=True, exist_ok=True) + + # Filename: _.json + filename = f"{account_id}_{today_str}.json" + filepath = acct_dir / filename + + with open(filepath, "w") as f: + json.dump(account_data, f, indent=2, cls=DateTimeEncoder) + + txn_count = len(account_data.get("transactions", [])) + rel_path = filepath.relative_to(output_path) + console.print(f" {account_name}: {txn_count} transactions -> {rel_path}") + + console.print(f"\nWrote {len(accounts)} account files to {output_dir}") + + if __name__ == "__main__": cli() From 286e5e464d083d5455425bde155906979da3fd3f Mon Sep 17 00:00:00 2001 From: robcohen Date: Wed, 17 Dec 2025 10:37:06 -0800 Subject: [PATCH 2/2] Add tests and documentation for fetch command - Add test_cli.py with 6 tests covering: - Directory structure creation - File naming convention - JSON content (account metadata + transactions) - Duplicate account name handling - Required options validation - Update README.md with fetch command documentation --- README.md | 49 ++++++++++ tests/test_cli.py | 236 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index 6f821f7..2613940 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,52 @@ cog.out( } ``` + +#### Fetch all accounts to separate files + +`simplefin fetch --output-dir DIRECTORY [--lookback-days INTEGER]` + +Fetches all accounts with their transactions and saves each account to a separate JSON file. Files are organized hierarchically by institution and account name: + +``` +output-dir/ + institution-domain/ + account-name/ + account-id_YYYY-MM-DD.json +``` + +This is useful for integration with tools like [beangulp](https://github.com/beancount/beangulp) that expect one file per account. + +``` +❯ simplefin fetch --output-dir ./simplefin-data --lookback-days 30 +Found 2 accounts + SimpleFIN Savings: 3 transactions -> beta-bridge.simplefin.org/SimpleFIN-Savings/Demo-Savings_2025-01-15.json + SimpleFIN Checking: 2 transactions -> beta-bridge.simplefin.org/SimpleFIN-Checking/Demo-Checking_2025-01-15.json + +Wrote 2 account files to ./simplefin-data +``` + +Each JSON file contains the full account metadata plus transactions: + +```json +{ + "org": { + "domain": "beta-bridge.simplefin.org", + "name": "SimpleFIN Demo" + }, + "id": "Demo Savings", + "name": "SimpleFIN Savings", + "currency": "USD", + "balance": "115525.50", + "balance-date": 1738368000, + "transactions": [ + { + "id": "1738382400", + "posted": "2025-02-01T12:00:00+00:00", + "amount": "-50.00", + "description": "Fishing bait", + "payee": "John's Fishin Shack" + } + ] +} +``` diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..12f2191 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,236 @@ +import datetime +import json +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from simplefin.cli import cli + + +@pytest.fixture +def mock_accounts(): + """Sample accounts response.""" + return [ + { + "org": { + "domain": "beta-bridge.simplefin.org", + "name": "SimpleFIN Demo", + }, + "id": "ACT-savings-123", + "name": "SimpleFIN Savings", + "currency": "USD", + "balance": "1000.00", + "balance-date": 1736553600, + }, + { + "org": { + "domain": "beta-bridge.simplefin.org", + "name": "SimpleFIN Demo", + }, + "id": "ACT-checking-456", + "name": "SimpleFIN Checking", + "currency": "USD", + "balance": "500.00", + "balance-date": 1736553600, + }, + ] + + +@pytest.fixture +def mock_transactions(): + """Sample transactions response.""" + return [ + { + "id": "TRN-001", + "posted": datetime.datetime(2025, 1, 10, 12, 0, 0), + "amount": "-50.00", + "description": "Test transaction", + "payee": "Test Payee", + }, + ] + + +class TestFetchCommand: + """Tests for the fetch CLI command.""" + + def test_fetch_creates_directory_structure(self, mock_accounts, mock_transactions): + """Test that fetch creates the correct directory structure.""" + runner = CliRunner() + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"SIMPLEFIN_ACCESS_URL": "https://mock"}): + with patch("simplefin.cli.SimpleFINClient") as MockClient: + mock_client = MockClient.return_value + mock_client.get_accounts.return_value = mock_accounts + mock_client.get_transactions.return_value = mock_transactions + + result = runner.invoke( + cli, + ["fetch", "--output-dir", tmpdir, "--lookback-days", "7"], + ) + + assert result.exit_code == 0 + assert "Found 2 accounts" in result.output + + # Check directory structure + inst_dir = Path(tmpdir) / "beta-bridge.simplefin.org" + assert inst_dir.exists() + + savings_dir = inst_dir / "SimpleFIN-Savings" + checking_dir = inst_dir / "SimpleFIN-Checking" + assert savings_dir.exists() + assert checking_dir.exists() + + def test_fetch_creates_json_files_with_correct_naming( + self, mock_accounts, mock_transactions + ): + """Test that JSON files are created with correct naming convention.""" + runner = CliRunner() + today = datetime.date.today().isoformat() + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"SIMPLEFIN_ACCESS_URL": "https://mock"}): + with patch("simplefin.cli.SimpleFINClient") as MockClient: + mock_client = MockClient.return_value + mock_client.get_accounts.return_value = mock_accounts + mock_client.get_transactions.return_value = mock_transactions + + result = runner.invoke( + cli, + ["fetch", "--output-dir", tmpdir, "--lookback-days", "7"], + ) + + assert result.exit_code == 0 + + # Check file naming + savings_file = ( + Path(tmpdir) + / "beta-bridge.simplefin.org" + / "SimpleFIN-Savings" + / f"ACT-savings-123_{today}.json" + ) + assert savings_file.exists() + + def test_fetch_json_contains_account_and_transactions( + self, mock_accounts, mock_transactions + ): + """Test that JSON files contain both account metadata and transactions.""" + runner = CliRunner() + today = datetime.date.today().isoformat() + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"SIMPLEFIN_ACCESS_URL": "https://mock"}): + with patch("simplefin.cli.SimpleFINClient") as MockClient: + mock_client = MockClient.return_value + mock_client.get_accounts.return_value = mock_accounts + mock_client.get_transactions.return_value = mock_transactions + + result = runner.invoke( + cli, + ["fetch", "--output-dir", tmpdir, "--lookback-days", "7"], + ) + + assert result.exit_code == 0 + + # Read and verify JSON content + savings_file = ( + Path(tmpdir) + / "beta-bridge.simplefin.org" + / "SimpleFIN-Savings" + / f"ACT-savings-123_{today}.json" + ) + + with open(savings_file) as f: + data = json.load(f) + + # Check account metadata + assert data["id"] == "ACT-savings-123" + assert data["name"] == "SimpleFIN Savings" + assert data["currency"] == "USD" + assert data["balance"] == "1000.00" + + # Check transactions were merged + assert "transactions" in data + assert len(data["transactions"]) == 1 + assert data["transactions"][0]["id"] == "TRN-001" + + def test_fetch_handles_duplicate_account_names(self, mock_transactions): + """Test that accounts with the same name get unique files.""" + runner = CliRunner() + today = datetime.date.today().isoformat() + + # Two accounts with the same name but different IDs + accounts_with_duplicates = [ + { + "org": {"domain": "example.com", "name": "Example Bank"}, + "id": "ACT-111", + "name": "Checking", + "currency": "USD", + "balance": "100.00", + "balance-date": 1736553600, + }, + { + "org": {"domain": "example.com", "name": "Example Bank"}, + "id": "ACT-222", + "name": "Checking", + "currency": "USD", + "balance": "200.00", + "balance-date": 1736553600, + }, + ] + + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"SIMPLEFIN_ACCESS_URL": "https://mock"}): + with patch("simplefin.cli.SimpleFINClient") as MockClient: + mock_client = MockClient.return_value + mock_client.get_accounts.return_value = accounts_with_duplicates + mock_client.get_transactions.return_value = [] + + result = runner.invoke( + cli, + ["fetch", "--output-dir", tmpdir, "--lookback-days", "7"], + ) + + assert result.exit_code == 0 + + # Both files should exist in the same directory + checking_dir = Path(tmpdir) / "example.com" / "Checking" + files = list(checking_dir.glob("*.json")) + assert len(files) == 2 + + # Files should have different account IDs in names + file_names = [f.name for f in files] + assert f"ACT-111_{today}.json" in file_names + assert f"ACT-222_{today}.json" in file_names + + def test_fetch_requires_output_dir(self): + """Test that --output-dir is required.""" + runner = CliRunner() + + with patch.dict(os.environ, {"SIMPLEFIN_ACCESS_URL": "https://mock"}): + result = runner.invoke(cli, ["fetch"]) + + assert result.exit_code != 0 + assert "Missing option '--output-dir'" in result.output + + def test_fetch_requires_access_url_env(self): + """Test that SIMPLEFIN_ACCESS_URL environment variable is required.""" + runner = CliRunner() + + with tempfile.TemporaryDirectory() as tmpdir: + # Ensure env var is not set + env = os.environ.copy() + env.pop("SIMPLEFIN_ACCESS_URL", None) + + with patch.dict(os.environ, env, clear=True): + result = runner.invoke( + cli, + ["fetch", "--output-dir", tmpdir], + ) + + # Should fail because no access URL + assert result.exit_code != 0