From 71ccbd0796ee931538f25816d63ab01d9f97dca6 Mon Sep 17 00:00:00 2001 From: Justin Mitchell Date: Mon, 13 Jul 2026 11:09:39 -0400 Subject: [PATCH 1/4] Add connector sync hub for Hardcover and Readwise Implements server-side fan-out sync to external reading services. Includes encrypted credential vault (AES-256-GCM), automatic book matching with title/author normalization, coalescing retry queue with exponential backoff, and management API for linking/unlinking services. Progress updates and highlights automatically sync to connected services. Requires TOKEN_ENC_KEY environment variable to enable. --- README.md | 1 + docs/API.md | 56 +++++++ docs/BUILD_STATUS.md | 86 +++++++++++ docs/design/hardcover-sync.md | 175 ++++++++++++++++++++++ docs/design/sync-hub.md | 258 +++++++++++++++++++++++++++++++++ migrations/0003_connectors.sql | 47 ++++++ src/app.ts | 10 +- src/connectors/fanout.ts | 54 +++++++ src/connectors/hardcover.ts | 192 ++++++++++++++++++++++++ src/connectors/matching.ts | 144 ++++++++++++++++++ src/connectors/queue.ts | 115 +++++++++++++++ src/connectors/readwise.ts | 154 ++++++++++++++++++++ src/connectors/registry.ts | 26 ++++ src/connectors/runner.ts | 139 ++++++++++++++++++ src/connectors/store.ts | 170 ++++++++++++++++++++++ src/connectors/types.ts | 99 +++++++++++++ src/crypto/secrets.ts | 76 ++++++++++ src/index.ts | 10 ++ src/routes/kosync.ts | 2 + src/routes/v1/clippings.ts | 26 ++++ src/routes/v1/connectors.ts | 182 +++++++++++++++++++++++ src/routes/v1/progress.ts | 2 + test/connectors.test.ts | 236 ++++++++++++++++++++++++++++++ test/helpers.ts | 9 +- test/matching.test.ts | 83 +++++++++++ test/queue.test.ts | 91 ++++++++++++ test/secrets.test.ts | 54 +++++++ 27 files changed, 2493 insertions(+), 4 deletions(-) create mode 100644 docs/BUILD_STATUS.md create mode 100644 docs/design/hardcover-sync.md create mode 100644 docs/design/sync-hub.md create mode 100644 migrations/0003_connectors.sql create mode 100644 src/connectors/fanout.ts create mode 100644 src/connectors/hardcover.ts create mode 100644 src/connectors/matching.ts create mode 100644 src/connectors/queue.ts create mode 100644 src/connectors/readwise.ts create mode 100644 src/connectors/registry.ts create mode 100644 src/connectors/runner.ts create mode 100644 src/connectors/store.ts create mode 100644 src/connectors/types.ts create mode 100644 src/crypto/secrets.ts create mode 100644 src/routes/v1/connectors.ts create mode 100644 test/connectors.test.ts create mode 100644 test/matching.test.ts create mode 100644 test/queue.test.ts create mode 100644 test/secrets.test.ts diff --git a/README.md b/README.md index f2b957e..5d8c2d0 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ DATABASE_PATH=./data/crosspoint.db PORT=8080 node dist/index.js | `DATABASE_PATH` | `/data/crosspoint.db` | SQLite file (parent dirs auto-created) | | `REGISTRATION_DISABLED` | `false` | Set `true` to lock down a private instance | | `AUTH_RATE_LIMIT_PER_MINUTE` | `30` | Per-IP limit on registration (0 disables) | +| `TOKEN_ENC_KEY` | _(unset)_ | Enables connectors (Hardcover/Readwise sync). 64 hex chars, a base64 32-byte key, or a ≥32-char passphrase. Encrypts stored connector credentials at rest; unset = connectors disabled. | ## Point your reader at it diff --git a/docs/API.md b/docs/API.md index 186928a..2a201ca 100644 --- a/docs/API.md +++ b/docs/API.md @@ -374,6 +374,62 @@ weighted by `pace_n`, `start_date` = earliest non-zero, `finished_date` = latest `GET /api/v1/documents` lists them. Most clients don't need this endpoint — progress-PUT `metadata` capture populates the same table. +### Connectors (master sync hub) + +Pair external services (Hardcover, Readwise, …) to the account; reading activity fans out to them +server-side. See docs/design/sync-hub.md. All under `/api/v1`, same auth headers. Requires the +server to have `TOKEN_ENC_KEY` set (credentials are encrypted at rest) — otherwise these endpoints +report `encryption: "disabled"` and linking returns 403. + +#### GET /api/v1/connectors + +Lists available connectors and this account's link status. + +```json +{ + "encryption": "enabled", + "connectors": [ + {"id": "hardcover", "name": "Hardcover", "tier": 1, "experimental": false, + "carries": ["progress", "finished"], "capabilities": {"read": false, "write": true}, + "credential_kind": "token", "linked": true, "status": "ok", "account": "julia", + "queue": {"pending": 0, "dead": 0}}, + {"id": "readwise", "name": "Readwise", "tier": 1, "experimental": false, + "carries": ["highlight"], "capabilities": {"read": true, "write": true}, + "credential_kind": "token", "linked": false, "status": null, "account": null} + ] +} +``` + +#### PUT /api/v1/connectors/{id} + +Link/re-link by validating and storing a credential. Body: `{"credential": { ... }}` — shape is +connector-specific (`{"token": "..."}` for Hardcover and Readwise). The server validates against the +service before storing; returns `400` if rejected. `{"id": "hardcover", "linked": true, +"account": "julia"}` on success. + +#### DELETE /api/v1/connectors/{id} + +Unlink; wipes the stored credential, all matches, and queued work. + +#### GET /api/v1/connectors/{id}/matches + +Lists resolved book matches (for a review UI): `{"connector": "hardcover", "matches": [{"document", +"external_id", "confidence", "source": "auto|manual|none", "query_used", "updated_at"}]}`. + +#### PUT /api/v1/connectors/{id}/matches/{document} + +Manually set a match (sticky — never auto-recomputed). Body `{"external_id": "42"}`, or +`{"external_id": null}` to mark "never sync this document". + +#### POST /api/v1/connectors/{id}/rematch/{document} + +Force (re)matching now; returns the resolved match or null. Preserves manual overrides. + +**Matching** is server-side from the document's title/author (the EPUB metadata the firmware sends — +so connectors need "Send Metadata" on). **Fan-out** is automatic: a progress PUT enqueues a +progress/finished event to write-connectors that carry it; a clippings PUT enqueues highlight events +to highlight-connectors (Readwise). A background worker delivers them with retry/backoff. + ### GET /healthz Unauthenticated. `{"status": "ok", "version": "0.1.0"}`. diff --git a/docs/BUILD_STATUS.md b/docs/BUILD_STATUS.md new file mode 100644 index 0000000..cf8a370 --- /dev/null +++ b/docs/BUILD_STATUS.md @@ -0,0 +1,86 @@ +# Build status — connector hub (overnight session) + +Snapshot of the master-sync-hub build. **Nothing committed or pushed.** Everything below is in the +working tree of `crosspoint-sync` for your review. + +## What's done and tested + +The full connector framework from `docs/design/sync-hub.md`, plus both Tier-1 connectors. +`npm test` = **74 passing** (was 40); `npx tsc --noEmit` clean; `npm run build` clean. + +### Framework +- **Encrypted secret vault** (`src/crypto/secrets.ts`) — AES-256-GCM, key from `TOKEN_ENC_KEY` + (hex / base64 / passphrase). Connectors disabled when unset; never stores plaintext. Tested + (roundtrip, tamper-detection, key formats, disabled mode). +- **Schema** (`migrations/0003_connectors.sql`) — `connector_accounts`, `connector_matches`, + `connector_queue`, keyed by `connector_id`. +- **Connector interface + registry** (`src/connectors/types.ts`, `registry.ts`) — one module per + service; `HttpTransport` is injectable so everything is testable without network. +- **Matcher** (`src/connectors/matching.ts`) — server-side title/author matching: normalize, + strip subtitles/series, `Last, First` handling, Jaccard scoring, auto-accept only on a clear + winner (same-book editions OK, title collisions rejected). Filename is a last resort only. + Heavily unit-tested. +- **Coalescing retry queue** (`src/connectors/queue.ts`) — one pending row per + (user, connector, document, coalesceKey); progress collapses to latest, highlights keyed per + clipping; exponential backoff, dead-letter after 8 tries. Tested. +- **Runner/worker** (`src/connectors/runner.ts`) — resolves match (cached/manual-sticky/auto), + pushes, handles reauth/backoff; `startQueueWorker` drains every 15s (started in `index.ts` only + when `TOKEN_ENC_KEY` is set). +- **Fan-out** (`src/connectors/fanout.ts`) — progress PUT → progress/finished events; clippings + PUT → highlight events. Wired into both kosync and v1 progress routes and the clippings route. + Best-effort, never blocks the request. +- **Management API** (`src/routes/v1/connectors.ts`) — list / link / unlink / list-matches / + manual-match / rematch. Documented in `docs/API.md`. Endpoint + fan-out tests in + `test/connectors.test.ts` (fake transport). + +### Connectors +- **Hardcover** (`src/connectors/hardcover.ts`, Tier 1) — token validate, search-based match, + status mutation (reading/read). Write-only, carries progress+finished. +- **Readwise** (`src/connectors/readwise.ts`, Tier 1) — token validate, highlight push (fan-out), + and `exportHighlights()` for the fan-in "aggregator hop" (Kindle highlights via Readwise). + Carries highlights. + +## ⚠️ Live-verify gates before enabling in production + +Both connectors are built against **documented** API shapes but not verified against live accounts +(I have no tokens, and shouldn't hit third-party APIs from here). Search the code for `GATE`. + +- **Hardcover** (beta API — highest risk): confirm against the live GraphQL explorer + (hardcover.app/account/api): the `me` query shape, `search` result shape (see `extractSearchHits`, + which defensively handles a few shapes), the `insert_user_book` mutation name/args, and the + **status ids** (`STATUS_READING`/`STATUS_READ` are guesses). Wire a recorded-fixture test once + confirmed so schema drift breaks CI. +- **Readwise** (stable public API — lower risk): confirm `POST /api/v2/highlights/` field names and + the `GET /api/v2/export/` cursor field (`nextPageCursor`). Mind the ~20 req/min limit on + create/export. + +## Not built (deliberately deferred) + +- **Fan-in wiring** — `exportHighlights()` exists but isn't hooked into a canonical-clippings + importer or a poll loop yet. That's the next chunk if you want Kindle-via-Readwise highlights + landing in the clippings store / on-device. +- **Tier 2/3 connectors** (Goodreads/StoryGraph cookie-replay, Kindle) — framework supports them + (`credentialKind: 'cookies'`, `experimental` flag) but none implemented; they need the CSRF + handshake + live capture work described in sync-hub.md, behind an experimental opt-in. +- **Web UI** — token paste / OAuth / match-review screen. The API is UI-ready; the UI is the + natural next project and the real prerequisite for non-technical pairing. +- **Bidirectional/fan-in merge rules** beyond the design notes. + +## To run locally with connectors on + +```sh +export TOKEN_ENC_KEY=$(openssl rand -hex 32) +DATABASE_PATH=./data/dev.db npm run dev +# GET /api/v1/connectors shows encryption: "enabled" and both connectors unlinked +``` + +Railway note: to enable connectors in prod, set `TOKEN_ENC_KEY` as a service variable (generate +once, keep it stable — rotating it invalidates all stored connector credentials). + +## Suggested review order + +1. `docs/design/sync-hub.md` (the plan) → `src/connectors/types.ts` (the shape). +2. `matching.ts` + `test/matching.test.ts` (the only tricky pure logic). +3. `runner.ts` + `queue.ts` + `test/queue.test.ts` (delivery semantics). +4. `hardcover.ts` / `readwise.ts` (check the GATEs against live docs). +5. `test/connectors.test.ts` (end-to-end link → sync → fan-out with a fake API). diff --git a/docs/design/hardcover-sync.md b/docs/design/hardcover-sync.md new file mode 100644 index 0000000..c5eb0d8 --- /dev/null +++ b/docs/design/hardcover-sync.md @@ -0,0 +1,175 @@ +# Design: Hardcover Sync Connector + +Status: **draft / not implemented** + +Forward reading activity from crosspoint-sync to [Hardcover](https://hardcover.app) so a user's +shelf reflects what they read on their e-reader — automatically. The connector lives entirely +server-side: devices keep speaking plain kosync and never talk to Hardcover. + +## Goals + +- When a user reads on any synced device, their Hardcover profile updates: book moves to + "Currently Reading" on first progress, progress percentage stays current, and the book flips to + "Read" (with a finish date) on completion. +- Zero firmware changes. Works for CrossPoint, CrossInk, and stock KOReader (once it ships the + metadata PR) alike. +- Hardcover being slow, down, or rate-limiting must never affect device sync. + +Non-goals (v1): syncing clippings/notes to Hardcover, importing Hardcover state back to devices, +StoryGraph (no public API), ratings/reviews. + +## Architecture + +``` +device --kosync PUT--> crosspoint-sync --enqueue--> forwarder --GraphQL--> Hardcover + | ^ + +-- matcher (title/author search, once per document) +``` + +Three pieces, all in this codebase: + +1. **Token link** — user attaches their Hardcover API token to their account. +2. **Matcher** — resolves our opaque document hash to a Hardcover book, using the + title/author/filename metadata we already capture. Runs once per `(user, document)`, result + cached; re-runs when metadata improves or the user overrides. +3. **Forwarder** — turns progress events into Hardcover GraphQL mutations, with a retry queue. + +### Why matching is server-side (decision) + +Title/author already arrive via the progress-PUT `metadata` object, so every existing client feeds +the matcher with no firmware release. Hardcover's search API (Typesense-backed) handles the fuzzy +matching; we only rank/accept results. Server-side logic can be improved and re-run against +historical documents at any time. Firmware-extracted ISBN was considered and rejected for the +critical path: it needs a firmware release to exist, and `dc:identifier` is missing or wrong in +enough real-world EPUBs (DRM-stripped, self-published, Calibre-converted) that we'd need the +fuzzy path anyway. ISBN remains a possible future confidence booster. + +## Hardcover API notes + +- Endpoint: `https://api.hardcover.app/v1/graphql`, header `Authorization: Bearer `. +- Tokens are user-generated (hardcover.app/account/api) and currently expire ~yearly; the API is + officially beta. Treat every request as fallible and every schema detail as subject to change. +- **Implementation gate:** before building the forwarder, verify against the live schema (the API + has a GraphQL explorer): search query shape, `user_books` status ids + (want-to-read / currently-reading / read), and the exact mutations for inserting/updating a + user book and its progress (book-level vs edition-level, pages vs percentage). Encode those in + one `src/connectors/hardcover.ts` module with a recorded-fixture test so schema drift breaks CI, + not production. +- Rate limits are modest (beta). The forwarder must coalesce: at most one progress mutation per + (user, book) per N minutes (default 15), always keeping only the latest value. Reading sessions + on e-ink produce sparse syncs anyway (sync is user-initiated over WiFi), so this is cheap. + +## Schema additions + +```sql +CREATE TABLE hardcover_accounts ( + user_id INTEGER PRIMARY KEY REFERENCES users(id), + token_enc TEXT NOT NULL, -- encrypted at rest (see Security) + enabled INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE hardcover_matches ( + user_id INTEGER NOT NULL REFERENCES users(id), + document TEXT NOT NULL, + book_id INTEGER, -- Hardcover book id (NULL = unmatched) + edition_id INTEGER, -- optional, when confidently known + confidence REAL NOT NULL DEFAULT 0, + source TEXT NOT NULL, -- 'auto' | 'manual' | 'none' + query_used TEXT, -- what we searched, for debugging/review UI + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, document) +); + +CREATE TABLE hardcover_queue ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + document TEXT NOT NULL, + kind TEXT NOT NULL, -- 'progress' | 'finished' + payload TEXT NOT NULL, -- JSON: percentage, timestamp, finished_date... + attempts INTEGER NOT NULL DEFAULT 0, + next_try_at INTEGER NOT NULL, + UNIQUE (user_id, document, kind) -- coalesce: newest payload replaces +); +``` + +## API additions (same x-auth headers) + +- `PUT /api/v1/connectors/hardcover` — `{token: "..."}`; server validates it with a cheap `me` + query before storing. `DELETE` unlinks and wipes the queue. +- `GET /api/v1/connectors/hardcover` — `{enabled, linked_at, matched: n, unmatched: n}`. +- `GET /api/v1/connectors/hardcover/matches` + `PUT .../matches/:document` — list and manually + set/override matches. Powers the future web-UI "review matches" screen; usable via curl until + then. Manual matches are sticky (`source = 'manual'`, never auto re-matched). + +Token entry realistically requires a browser, not an e-ink keyboard — this lands with (or just +before) the first web UI. The endpoints don't depend on the UI, though. + +## Matching algorithm + +Input: the EPUB's **`documents.title` + `documents.author`** — the real signal, which the firmware +already extracts and sends in the progress `metadata` object. This is the primary/expected path. +Filename is only a last resort for a title-less EPUB, and we don't guess `Title - Author` vs +`Author - Title` order (drop separators, use the whole string as a fuzzy query). Filename can't +rescue the no-metadata case — title/author and filename ship in the *same* metadata object — so +**matching effectively requires "Send Metadata" on**, which is the reason to default that toggle on +for connector users. + +1. Normalize: strip subtitle after `:`, series suffixes like `(Book 2)`, diacritics folded, + `Last, First` → `First Last`. +2. Query Hardcover search with `title author`. +3. Score the top results: normalized-title similarity + author-name overlap. Accept the top hit + when it clears a threshold **and** clearly beats the runner-up (title collisions like + "Circe" resolve on author; same-title-same-author editions are all correct at book level, so + ambiguity between them is acceptable — take the most popular edition). +4. Store the result either way (`source: 'auto'` or `'none'` with the query kept for review). + +Re-match triggers: metadata for the document changes (e.g. the user flips Send Metadata on after +the fact), or a manual re-match request. Never re-match over `source = 'manual'`. + +Documents with no metadata at all simply stay unmatched — which is also the user-facing nudge to +enable the Send Metadata toggle. + +## Forwarding rules + +On progress PUT for a linked user with a matched document, upsert into the queue: + +- percentage > 0 → `progress` event: ensure user_book exists with status currently-reading, + update progress percentage. First event for a book is what shelves it. +- completion → `finished` event: status read + finish date. Completion = percentage ≥ 0.98, or — + once CrossInk stats sync ships — `isCompleted` / non-zero `finished_date` from + `stats_device_book` (the stronger signal wins; stats-based finish dates are real dates, the + percentage heuristic uses the sync timestamp). + +A small worker drains the queue (setInterval in the Node process — no external infra), with +exponential backoff on failure and a dead-letter state after ~a week of retries. `401` from +Hardcover (expired token) disables the link and surfaces in the status endpoint rather than +retrying forever. + +Kosync GET/downstream sync is never blocked or delayed by any of this. + +## Security + +- Hardcover tokens are bearer credentials to a third-party account: encrypt at rest + (AES-256-GCM with a key from a `TOKEN_ENC_KEY` env var), never log them, redact in errors. + Self-hosters without `TOKEN_ENC_KEY` set get the connector disabled, not plaintext storage. +- Outbound requests go only to the Hardcover endpoint; no user-controlled URLs (no SSRF surface). + +## Rollout + +1. Schema + token link + matcher, with `GET .../matches` for inspection. Verify match quality on + real libraries via curl before any forwarding exists. +2. Forwarder for progress + finished, behind per-user `enabled`. +3. Web UI: token entry + review-matches screen. +4. Later: ISBN confidence boost, richer status mapping (owned/DNF), maybe Bookwyrm/Calibre-Web + via the same connector pattern. + +## Open questions + +- Exact Hardcover mutation set and status ids (resolve at the implementation gate above). +- Should percentage-completion threshold be user-configurable? (Default 0.98; back-matter skews + short books.) +- Multi-device: progress forwarding uses the newest-across-devices row (same rule as kosync GET), + so no per-device fan-out — confirm that's the desired UX for people who read the same book on + two devices at different points. diff --git a/docs/design/sync-hub.md b/docs/design/sync-hub.md new file mode 100644 index 0000000..b0a3203 --- /dev/null +++ b/docs/design/sync-hub.md @@ -0,0 +1,258 @@ +# Design: Sync Hub & Connector Framework + +Status: **draft / not implemented** — architectural direction, not a commitment to ship every +connector below. Read the feasibility tiers before scoping anything. + +## Idea + +A crosspoint-sync account becomes a **master sync identity**. Reading state (progress, finished +status, and later ratings/clippings) is captured once — from any device over kosync — and +fanned out to whichever external services the user has *paired*: Hardcover, Goodreads, StoryGraph, +Kindle/Whispersync, Audible, etc. All pairing and forwarding happens server-side, so there are +**no firmware changes** for any of it. + +[hardcover-sync.md](hardcover-sync.md) is the reference connector; this doc generalizes that +pattern to N connectors and defines what's actually buildable. + +## The hub, conceptually + +``` + crosspoint-sync account (master identity) + | + +------------------+------------+------------+------------------+ + | | | | + device sync Hardcover Goodreads Kindle / Audible + (kosync, canonical) connector connector connector + | | | | + progress/stats <--- sync graph: canonical store + per-connector adapters ---> +``` + +- **Canonical store** = the tables we already have (`progress`, `documents`, stats, bookmarks, + clippings). This is the source of truth and the lowest common denominator. +- **Connectors** are adapters. Each declares: which credential type it needs, whether it's + read/write/bidirectional, how it matches our document hash to its own book identity, and how it + maps our state to its state. +- **Fan-out** is triggered by a change to the canonical store (a progress PUT). **Fan-in** + (bidirectional connectors pushing external changes back into the canonical store) is a poll or + webhook per connector — only some connectors can do it. + +## Connector interface (shared shape) + +Every connector, regardless of service, is one module implementing: + +```ts +interface Connector { + id: string; // 'hardcover' | 'goodreads' | ... + capabilities: { read: boolean; write: boolean }; // fan-in / fan-out support + credentialKind: 'oauth' | 'token' | 'cookies'; // never 'password' — see auth models + validate(cred): Promise<{ ok: boolean; account?: string }>; + match(doc: DocumentMeta): Promise; // hash -> external book id + pushProgress(cred, match, ev): Promise; // fan-out (write connectors) + pullChanges?(cred, since): Promise; // fan-in (read connectors) +} +``` + +This reuses the Hardcover design's three-part structure (token vault, match table, retry queue) +for all connectors. Adding a service = adding one module + one row in a connector registry; the +hub, queue, matcher-runner, and web UI are written once. Matching is **always server-side**, using +the title/author/filename metadata the devices already send (see hardcover-sync.md — the same +"why server-side" reasoning applies universally). + +Shared tables generalize the Hardcover ones: `connector_accounts(user_id, connector_id, +cred_enc, enabled, ...)`, `connector_matches(user_id, connector_id, document, external_id, +confidence, source, ...)`, `connector_queue(...)`. One schema, keyed by `connector_id`. + +## Auth models — the key insight: cookie-replay ≠ storing passwords + +The connector landscape is uneven, but the dividing line is **auth mechanism**, not "has an API." +Three models, best to worst posture: + +1. **Scoped token / OAuth (Tier 1).** Service issues a per-user token with the user's consent. + Revocable, sometimes scopeable. Best case. (Hardcover.) +2. **Cookie-replay (Tier 2).** User is already logged in on the service in their browser; they + paste their session cookies (or a browser extension harvests just those cookies). The connector + replays them against the service's own **web endpoints**. **The server never sees the + password.** Because the cookie is captured *post-login*, this also sidesteps 2FA/CAPTCHA + entirely. This is the standard modern pattern — it's how Readwise's Kindle sync works + ("Readwise couldn't access your Amazon password even if we wanted to"). The tradeoff isn't + credential exposure — it's that session cookies are unscopeable full-session secrets, expire and + need re-pasting, and depend on undocumented web endpoints that can change. +3. **Stored password / headless login (avoid).** Server holds the actual username+password and + drives a login. Highest risk; breaks on 2FA. **We do not build this for any connector.** + +Cookie-replay is what makes Goodreads/StoryGraph/Kindle viable at all now — and viable *without* +holding anyone's password. It's still ToS-gray and brittle, so those connectors ship behind an +explicit "experimental, may break, unofficial" opt-in — but "we store your password" was never +the actual requirement, and I was wrong to frame it that way earlier. + +### Prefer the aggregator hop over direct scraping + +When a **sanctioned aggregator already ingests a hard target**, route through it instead of +scraping the target ourselves. The prime example: **Readwise** already pulls Kindle highlights via +its own browser extension (the ToS-gray Amazon work is *their* responsibility, done with the user's +explicit install), and exposes them through an official token API. So for Kindle *highlights*, a +Readwise connector (Tier 1, no scraping) beats a direct Kindle cookie-replay connector (Tier 3, +TLS-fingerprint fight). Always ask "is there a legit API that already has this data?" before +building a scraper. (This doesn't cover Kindle reading-*progress* — Readwise doesn't have it — so +the direct-Kindle spike still stands for progress specifically.) + +## Feasibility tiers — READ THIS BEFORE SCOPING + +### Tier 1 — Sanctioned public API. Buildable, durable. + +- **Hardcover** — public GraphQL API (beta), per-user bearer tokens. Write + partial read. + Carries *progress + shelves + rating*. See hardcover-sync.md. **Ship first.** +- **Readwise** — official REST API, per-user access token (`Authorization: Token `, issued at + readwise.io/access_token). Carries *highlights/notes only* — no reading progress or shelves. + **Bidirectional and the safest connector we have**, because it's a fully sanctioned token API + (no cookies, no scraping, user-revocable). Two high-value uses: + - **Fan-out:** push CrossInk clippings into Readwise (`POST /api/v2/highlights/`, batched) — from + there they flow to the user's whole highlight ecosystem (Notion, Obsidian, Readwise reviews). + This is the cleanest way to make on-device highlights useful anywhere. + - **Fan-in / "Readwise as the hop":** pull highlights *out* of Readwise + (`GET /api/v2/export/?updatedAfter=`) into our canonical clippings store. Since Readwise's own + browser extension already ingests **Kindle** (and Apple Books, Instapaper, …) highlights the + sanctioned way, this gets us Kindle highlights **without us ever scraping Amazon** — we let + Readwise do the Amazon work and read from their clean API. This is the safer alternative to a + direct Kindle connector for the *highlights* use case. + + Caveats: highlights-only (does **not** solve Kindle reading-*progress* sync — that's still the + Tier-3 spike below), requires a paid Readwise subscription, and create/export endpoints are + rate-limited (~20 req/min — batch accordingly). Book identity in Readwise is title/author, which + matches our metadata model. Verify the exact v2 schema at implementation (same gate as Hardcover). + +### Tier 2 — No official API, but cookie-replay works. Experimental, no passwords stored. + +- **Goodreads** — API keys dead since **Dec 2020**. The well-known Calibre "Goodreads Sync" plugin + still works only because it ships the author's own *grandfathered* OAuth key, shared across its + whole userbase — **we can't and shouldn't reuse that** (a new app can't register a key; borrowing + theirs invites revocation + ToS violation). The viable path for us is **cookie-replay against the + Goodreads web endpoints** (`_session_id2` session cookie + Rails `authenticity_token` CSRF + scraped from page HTML). This can *write*: shelve, rate, set read date, and update reading + progress. See "Goodreads write-path" below. +- **StoryGraph** — no public API, but cookie-replay is proven: the `storygraph.koplugin` KOReader + plugin already writes progress %, status, and auto-marks Read using two cookies + (`_story_graph_session` + `remember_user_token`) + CSRF. **Note the competitive context: Kobo + shipped *native* StoryGraph sync in June 2026** — so StoryGraph is where the e-reader crowd is + heading, which raises the value of us having it and lowers the novelty of Goodreads. + + Verdict: both buildable via cookie-replay with no password storage. Gate behind an explicit + experimental opt-in; expect breakage on site changes and cookie expiry. Prefer official APIs if + they ever appear (StoryGraph has discussed one). + +### Tier 3 — Cookie-replay possible, but higher blast radius / harder. Spike, don't commit. + +- **Amazon Kindle (`read.amazon.com`)** — cookie-replay is proven (`Xetera/kindle-api` reads + library + reading-progress % using `at-main`/`sess-at-main`/`x-main`/`ubid-main`/`session-id` + cookies, valid ~1 year). Two real obstacles beyond Tier 2: (a) **Amazon added TLS fingerprinting + in July 2023**, so a naive server fetch is blocked — you need a browser-mimicking TLS client + (bogdanfinn/tls-client style) or you route through the user's browser via an extension; (b) an + Amazon session cookie is higher blast radius than a Goodreads one (same account as payments, + though scoped to the `read.amazon.com` subdomain in Readwise's model). Whispersync itself (the + device progress protocol) remains private with no endpoint; what's reachable is the Cloud Reader + progress % and the `/notebook` highlights. So "bidirectional Kindle progress" is partially real + (read progress %, write via the same web surface) but engineering-heavy and ToS-gray. +- **Audible** — only a community reverse-engineered API; audiobook position ≠ ebook position + (needs a timestamp↔percentage model). Lower priority. + + Verdict: technically reachable via the same cookie-replay pattern (no password storage), but the + Amazon TLS-fingerprinting workaround and the larger credential blast radius make this a research + spike gated on a security review — not a committed v1 feature. **For highlights specifically, + prefer the Readwise hop (Tier 1) over building this at all.** A direct Kindle connector is only + justified by reading-*progress* sync, which Readwise can't provide. The framework accommodates + it; we don't rush it. + +## What this means for the build + +1. **Build the hub + connector framework** (generalized tables, registry, queue worker, matcher + runner, web-UI pairing screen). Durable regardless of which connectors follow. +2. **Ship Hardcover + Readwise** as the first Tier-1 connectors (token APIs, no extension needed). +3. **Build the browser extension** — the decided, shared credential-capture path for every + cookie-based connector. It's the prerequisite for all of Tier 2/3, so it comes before them. +4. **Gate Tier 2** (Goodreads/StoryGraph, via the extension) behind an explicit experimental opt-in + if there's demand; expect maintenance cost and breakage. Revisit if official APIs appear. +5. **Do not build Tier 3 credential storage.** Track the Amazon/Audible landscape; if an official + API or a legal-reviewed narrow importer becomes viable, the framework already accommodates it. + +## Cookie-replay mechanics (Tier 2/3) + +Shared shape for every cookie-replay connector: + +1. **Credential capture — via a first-party browser extension (decided).** The extension is the + committed capture path for all cookie-based connectors: the user logs into the service in their + own browser, and the extension harvests *only* the specific cookie names below and POSTs them to + us. This is strictly better than paste-the-cookie (no DevTools spelunking, no accidental + over-sharing, matches Readwise's proven UX) and never touches the login page or password. + Manual cookie paste stays as a no-extension fallback only. We store the harvested bundle + encrypted, treated as password-equivalent. Cookie sets per service: + - Goodreads: `_session_id2` (+ `ccsid`) + - StoryGraph: `_story_graph_session` + `remember_user_token` + - Kindle: `at-main`, `sess-at-main`, `x-main`, `ubid-main`, `session-id` +2. **CSRF handshake (Rails sites: Goodreads, StoryGraph).** Before any write, GET an authenticated + HTML page, scrape `` (or the hidden `authenticity_token` input), and + send it as `X-CSRF-Token` (AJAX) or an `authenticity_token` form field (form POST), alongside + the session cookie. +3. **Write** to the service's web endpoint (below). +4. **Expiry handling.** On a redirect-to-login / 401 / signup-page-HTML response, mark the + connector `needs_reauth` and surface it in the status endpoint — never retry-loop a dead + session. Kindle cookies last ~1 year; Goodreads/StoryGraph session cookies are shorter. + +### Goodreads write-path reference + +We can't use the Goodreads API (dead keys), but the **field names** from the still-maintained +Calibre "Goodreads Sync" plugin (`kiwidude68/calibre_plugins → goodreads_sync/core.py`) are the +best-documented reference, because Goodreads' web forms and its old API share parameter naming. +Capture the *current* web/AJAX paths from DevTools while shelving/rating/updating on goodreads.com, +then map onto these known field shapes: + +| Operation | Params (urlencoded) | +|---|---| +| Add/remove shelf | `name=`, `book_id=` (+ `a=remove`) | +| Rating + read date | `review[rating]`, `review[read_at]=YYYY-MM-DD`, `review[review]` | +| Reading progress | `user_status[book_id]`, `user_status[percent]` (or `[page]`), `user_status[body]` | + +Book matching is title/author search (server-side, as everywhere in this doc): Goodreads +`/search/search.xml`-style query, or ISBN if ever available. **Caveat:** the `user_status` progress +update is historically the flakiest Goodreads endpoint (intermittent 401s, "success but no visible +change") — treat progress as best-effort and rely on shelf + read-date as the durable signal. + +StoryGraph writes follow the same GET-CSRF-then-POST shape; `storygraph.koplugin` is the reference +for the current status/progress endpoints. + +## Bidirectional sync (fan-in) notes + +Only connectors with a read capability can push external changes back. The merge rule extends the +existing kosync "newest wins across devices" to "newest wins across sources" — every canonical +event carries a source and timestamp; the hub applies the most recent and re-fans-out to the +others, with loop suppression (don't echo a change back to the source that produced it). This is +straightforward for Tier 1 and mostly irrelevant for Tier 2/3, which are write-mostly or +unavailable. + +## Security posture (applies to all connectors) + +- **We never store passwords.** Tier 2/3 use cookie-replay: the server holds session cookies, not + credentials, and never sees the plaintext password (nor 2FA, since cookies are post-login). +- All external credentials (tokens and cookie bundles) encrypted at rest (`AES-256-GCM`, key from + `TOKEN_ENC_KEY`); connector disabled if the key is unset (self-host default). Never logged, + redacted in errors. +- **Cookie bundles are unscopeable full-session secrets** — treat them as sensitive as passwords + even though they aren't passwords: minimal retention, encrypt, and lean on the user's natural + revocation levers (password change / "log out everywhere" invalidates them). A leaked Amazon + cookie is higher blast radius than a Goodreads one; that's the extra weight on Tier 3. +- **The browser extension is the decided capture model** for cookie connectors (harvest only the + named cookies, like Readwise) — strictly better privacy than paste-the-cookie and than ever + touching the login page. Paste-cookie is the no-extension fallback. The extension is a shared + dependency of every Tier 2/3 connector, so it's the first thing to build before any of them ship. +- Outbound requests restricted to each connector's known hosts (no user-supplied URLs; no SSRF). + Kindle additionally needs a browser-fingerprint-matching TLS client to get past Amazon's 2023 + TLS fingerprinting. + +## Open questions + +- Web UI is a hard prerequisite for pairing anything (OAuth redirects, token paste) — this is the + forcing function that makes the web UI real. Sequence accordingly. +- Per-connector completion/rating mapping differs (star scales, half-stars, DNF states) — define a + canonical rating model once, adapt per connector. +- Do we expose the master account as its own login (email/password/OAuth) distinct from the kosync + credential, so a user can manage pairings without device credentials? Likely yes, once the web + UI exists. diff --git a/migrations/0003_connectors.sql b/migrations/0003_connectors.sql new file mode 100644 index 0000000..563f94a --- /dev/null +++ b/migrations/0003_connectors.sql @@ -0,0 +1,47 @@ +-- Master-sync-hub connector framework. One row set per external service +-- (Hardcover, Readwise, ...), keyed by connector_id. See docs/design/sync-hub.md. + +CREATE TABLE connector_accounts ( + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, -- 'hardcover' | 'readwise' | ... + cred_enc TEXT NOT NULL, -- AES-256-GCM encrypted credential JSON + account_label TEXT, -- display name/email from the service, if known + status TEXT NOT NULL DEFAULT 'ok', -- 'ok' | 'needs_reauth' | 'error' + enabled INTEGER NOT NULL DEFAULT 1, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, connector_id) +); + +CREATE TABLE connector_matches ( + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, + document TEXT NOT NULL, + external_id TEXT, -- service book id (NULL = unmatched) + external_edition TEXT, -- optional edition id + confidence REAL NOT NULL DEFAULT 0, + source TEXT NOT NULL DEFAULT 'none', -- 'auto' | 'manual' | 'none' + query_used TEXT, -- what we searched, for the review UI + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, connector_id, document) +); +CREATE INDEX idx_conn_matches_doc ON connector_matches(user_id, document); + +CREATE TABLE connector_queue ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, + document TEXT NOT NULL, + kind TEXT NOT NULL, -- 'progress' | 'finished' | 'highlight' + payload TEXT NOT NULL, -- JSON event body + attempts INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', -- 'pending' | 'done' | 'dead' + next_try_at INTEGER NOT NULL, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + -- Coalesce: a newer event of the same kind for the same book replaces the pending one. + UNIQUE (user_id, connector_id, document, kind) +); +CREATE INDEX idx_conn_queue_ready ON connector_queue(status, next_try_at); diff --git a/src/app.ts b/src/app.ts index 31f4274..a449e0f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,11 +8,18 @@ import { bookmarkRoutes } from './routes/v1/bookmarks.js'; import { clippingRoutes } from './routes/v1/clippings.js'; import { statsRoutes } from './routes/v1/stats.js'; import { documentRoutes } from './routes/v1/documents.js'; +import { connectorRoutes } from './routes/v1/connectors.js'; +import type { HttpTransport } from './connectors/types.js'; // Injected at build time via package.json; read lazily to keep this file dependency-free. export const VERSION = process.env.npm_package_version ?? '0.1.0'; -export function createApp(db: DB, config: Config): Hono { +export interface AppOptions { + /** Override the connector HTTP transport (tests inject a fake). */ + connectorTransport?: HttpTransport; +} + +export function createApp(db: DB, config: Config, opts: AppOptions = {}): Hono { const app = new Hono(); app.get('/healthz', (c) => c.json({ status: 'ok', version: VERSION })); @@ -29,6 +36,7 @@ export function createApp(db: DB, config: Config): Hono { v1.route('/', clippingRoutes(db)); v1.route('/', statsRoutes(db)); v1.route('/', documentRoutes(db)); + v1.route('/', connectorRoutes(db, opts.connectorTransport)); app.route('/api/v1', v1); return app; diff --git a/src/connectors/fanout.ts b/src/connectors/fanout.ts new file mode 100644 index 0000000..e8d2dff --- /dev/null +++ b/src/connectors/fanout.ts @@ -0,0 +1,54 @@ +import type { DB } from '../db/db.js'; +import { secretsEnabled } from '../crypto/secrets.js'; +import { getConnector } from './registry.js'; +import { enqueue } from './queue.js'; +import { activeConnectorIds } from './store.js'; +import type { OutboundEvent } from './types.js'; + +/** + * Enqueue canonical reading events to every linked connector that carries the + * event's kind. Best-effort and synchronous-but-cheap (DB inserts only); the + * queue worker does the network I/O. Never throws into the request path. + */ +function fanOut(db: DB, userId: number, ev: OutboundEvent, coalesceKey?: string): void { + if (!secretsEnabled()) return; + try { + for (const connectorId of activeConnectorIds(db, userId)) { + const conn = getConnector(connectorId); + if (!conn || !conn.capabilities.write || !conn.carries.includes(ev.kind)) continue; + enqueue(db, userId, connectorId, ev, coalesceKey); + } + } catch (err) { + console.error( + JSON.stringify({ msg: 'fanout enqueue failed', error: err instanceof Error ? err.message : String(err) }) + ); + } +} + +export function fanOutProgress( + db: DB, + userId: number, + document: string, + percentage: number, + timestamp: number +): void { + const finished = percentage >= 0.98; + fanOut(db, userId, { + kind: finished ? 'finished' : 'progress', + document, + percentage, + timestamp, + }); +} + +export function fanOutHighlight( + db: DB, + userId: number, + document: string, + clippingId: string, + h: NonNullable, + timestamp: number +): void { + // Per-clipping coalesce key so distinct highlights on one book each queue. + fanOut(db, userId, { kind: 'highlight', document, timestamp, highlight: h }, `highlight:${clippingId}`); +} diff --git a/src/connectors/hardcover.ts b/src/connectors/hardcover.ts new file mode 100644 index 0000000..3f88c89 --- /dev/null +++ b/src/connectors/hardcover.ts @@ -0,0 +1,192 @@ +import { decideMatch, extractTitleAuthor, type Candidate } from './matching.js'; +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * Hardcover connector (Tier 1). Public GraphQL API, per-user bearer token. + * Carries reading progress + shelf status. + * + * !!! LIVE-VERIFY GATE !!! + * Hardcover's API is beta. The GraphQL operations below (field names, the + * `me`/search shapes, and the user_book mutation names/status ids) are modeled + * from the documented schema but MUST be checked against the live GraphQL + * explorer at https://hardcover.app/account/api before enabling in production. + * Every network call is funneled through gql() so the exact queries live in one + * place and are covered by fixture tests. Search the file for GATE to find each + * spot that needs confirmation. + */ + +const ENDPOINT = 'https://api.hardcover.app/v1/graphql'; + +// GATE: confirm Hardcover's user_book status ids (want-to-read/reading/read). +const STATUS_READING = 2; +const STATUS_READ = 3; + +interface HardcoverCred extends Credential { + token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as HardcoverCred).token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing hardcover token'); + return t; +} + +async function gql( + http: HttpTransport, + token: string, + query: string, + variables: Record +): Promise<{ data?: any; errors?: { message: string }[]; status: number }> { + const res = await http(ENDPOINT, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: token.startsWith('Bearer ') ? token : `Bearer ${token}`, + }, + body: JSON.stringify({ query, variables }), + }); + if (res.status === 401 || res.status === 403) { + return { status: res.status, errors: [{ message: 'unauthorized' }] }; + } + let body: any = {}; + try { + body = await res.json(); + } catch { + body = {}; + } + return { status: res.status, data: body.data, errors: body.errors }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + // GATE: confirm the `me` query shape. + const r = await gql(http, token, `query { me { username } }`, {}); + if (r.status === 401 || r.status === 403) return { ok: false, error: 'invalid token' }; + if (r.errors?.length) return { ok: false, error: r.errors[0].message }; + const username = r.data?.me?.[0]?.username ?? r.data?.me?.username; + return { ok: true, accountLabel: username ?? undefined }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +async function match( + cred: Credential, + doc: DocumentMeta, + http: HttpTransport +): Promise { + const ta = extractTitleAuthor(doc); + if (!ta) return null; + const token = tokenOf(cred); + const q = `${ta.title} ${ta.author}`.trim(); + // GATE: confirm Hardcover's search query name and result shape. + const r = await gql( + http, + token, + `query Search($q: String!) { + search(query: $q, query_type: "Book", per_page: 10) { + results + } + }`, + { q } + ); + if (r.errors?.length || !r.data) return null; + const hits = extractSearchHits(r.data); + if (hits.length === 0) return null; + const decision = decideMatch(ta.title, ta.author, hits); + if (!decision.accepted || !decision.best) return null; + return { + externalId: decision.best.externalId, + confidence: decision.best.score, + queryUsed: q, + }; +} + +/** GATE: adapt to the real search payload. Handles a couple of plausible shapes. */ +export function extractSearchHits(data: any): Candidate[] { + const raw = + data?.search?.results?.hits ?? + data?.search?.results ?? + data?.search ?? + []; + const arr = Array.isArray(raw) ? raw : Array.isArray(raw?.hits) ? raw.hits : []; + const out: Candidate[] = []; + for (const h of arr) { + const doc = h?.document ?? h; + const id = doc?.id ?? doc?.book_id; + const title = doc?.title; + if (id == null || typeof title !== 'string') continue; + const author = + doc?.author_names?.[0] ?? + doc?.contributions?.[0]?.author?.name ?? + doc?.author ?? + undefined; + out.push({ + externalId: String(id), + title, + author, + popularity: typeof doc?.users_count === 'number' ? doc.users_count : undefined, + }); + } + return out; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const token = tokenOf(cred); + const bookId = Number(m.externalId); + const finished = ev.kind === 'finished' || (ev.percentage ?? 0) >= 0.999; + const status = finished ? STATUS_READ : STATUS_READING; + + // GATE: confirm the upsert mutation name/args. Hardcover uses an + // insert_user_book / update_user_book pattern keyed by book + status. + const mutation = ` + mutation SetStatus($bookId: Int!, $status: Int!) { + insert_user_book(object: { book_id: $bookId, status_id: $status }) { + id + } + }`; + const r = await gql(http, token, mutation, { bookId, status }); + + if (r.status === 401 || r.status === 403) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (r.status === 429) { + return { ok: false, retryable: true, error: 'rate limited' }; + } + if (r.errors?.length) { + // GraphQL validation errors won't fix themselves on retry. + return { ok: false, retryable: false, error: r.errors[0].message }; + } + if (r.status >= 500) { + return { ok: false, retryable: true, error: `server ${r.status}` }; + } + return { ok: true }; +} + +export const hardcoverConnector: Connector = { + id: 'hardcover', + displayName: 'Hardcover', + tier: 1, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'token', + experimental: false, + validate, + match, + push, +}; diff --git a/src/connectors/matching.ts b/src/connectors/matching.ts new file mode 100644 index 0000000..2de3b0e --- /dev/null +++ b/src/connectors/matching.ts @@ -0,0 +1,144 @@ +import type { DocumentMeta } from './types.js'; + +/** + * Shared, connector-agnostic book-matching helpers. Connectors call their own + * search API, then use scoreCandidate() to rank results against the document's + * title/author. Pure functions — unit-tested independently of any network. + */ + +/** Fold diacritics, lowercase, drop punctuation, collapse whitespace. */ +export function normalizeText(s: string): string { + return s + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') // combining marks + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** Strip subtitle after a colon and trailing "(Series, Book 2)" style suffixes. */ +export function coreTitle(title: string): string { + let t = title.replace(/\s*[:—-]\s.*$/, ''); // subtitle + t = t.replace(/\s*\((?:[^)]*\b(?:book|vol|volume|part|no)\b[^)]*)\)\s*$/i, ''); + return t.trim(); +} + +/** "Last, First" -> "First Last"; also normalizes. */ +export function normalizeAuthor(author: string): string { + const trimmed = author.trim(); + const comma = trimmed.indexOf(','); + if (comma > 0 && trimmed.indexOf(',') === trimmed.lastIndexOf(',')) { + const last = trimmed.slice(0, comma).trim(); + const first = trimmed.slice(comma + 1).trim(); + return normalizeText(`${first} ${last}`); + } + return normalizeText(trimmed); +} + +/** + * Derive {title, author} for book matching. The EPUB's own title/author (which + * the firmware extracts and sends in the progress `metadata` object) is the real + * signal — this is the primary and expected path. + * + * Filename is only a last resort for the rare title-less case (a malformed EPUB + * whose getTitle() was empty). Note it can't rescue the "no metadata at all" + * case: filename ships in the same metadata object as title/author, so if we + * lack title we usually lack filename too. We deliberately do NOT guess + * "Title - Author" vs "Author - Title" ordering — we drop the separators and let + * the whole string be a fuzzy search query, which search engines handle fine. + */ +export function extractTitleAuthor(doc: DocumentMeta): { title: string; author: string } | null { + if (doc.title) { + return { title: doc.title, author: doc.author ?? '' }; + } + if (doc.filename) { + const base = doc.filename.replace(/\.[a-z0-9]+$/i, '').replace(/\s*-\s*/g, ' ').trim(); + return base ? { title: base, author: '' } : null; + } + return null; +} + +/** Token set overlap (Jaccard) of two normalized strings. */ +function tokenOverlap(a: string, b: string): number { + const sa = new Set(a.split(' ').filter(Boolean)); + const sb = new Set(b.split(' ').filter(Boolean)); + if (sa.size === 0 || sb.size === 0) return 0; + let inter = 0; + for (const t of sa) if (sb.has(t)) inter++; + return inter / (sa.size + sb.size - inter); +} + +export interface Candidate { + externalId: string; + title: string; + author?: string; + /** Optional popularity/rank signal (higher = more popular), used as a tiebreak. */ + popularity?: number; +} + +export interface ScoredCandidate extends Candidate { + score: number; +} + +/** + * Score a candidate against the wanted title/author. Title similarity dominates; + * author overlap is a strong secondary that disambiguates same-title collisions. + */ +export function scoreCandidate( + wantTitle: string, + wantAuthor: string, + cand: Candidate +): number { + const wt = normalizeText(coreTitle(wantTitle)); + const ct = normalizeText(coreTitle(cand.title)); + const titleScore = wt && ct ? tokenOverlap(wt, ct) : 0; + + const wa = wantAuthor ? normalizeAuthor(wantAuthor) : ''; + const ca = cand.author ? normalizeAuthor(cand.author) : ''; + const authorScore = wa && ca ? tokenOverlap(wa, ca) : 0; + + // No author info on either side: rely on title alone (capped so it can't + // clear a high threshold on title-only, since title collisions are common). + if (!wa || !ca) return titleScore * 0.85; + return titleScore * 0.7 + authorScore * 0.3; +} + +export interface MatchDecision { + best: ScoredCandidate | null; + accepted: boolean; +} + +/** + * Rank candidates and decide whether to auto-accept: the top must clear + * `threshold` AND beat the runner-up by `margin` (unless the runner-up is the + * same book — same normalized title+author — in which case ambiguity between + * editions is fine and we take the more popular one). + */ +export function decideMatch( + wantTitle: string, + wantAuthor: string, + candidates: Candidate[], + opts: { threshold?: number; margin?: number } = {} +): MatchDecision { + const threshold = opts.threshold ?? 0.6; + const margin = opts.margin ?? 0.15; + if (candidates.length === 0) return { best: null, accepted: false }; + + const scored: ScoredCandidate[] = candidates + .map((c) => ({ ...c, score: scoreCandidate(wantTitle, wantAuthor, c) })) + .sort((a, b) => b.score - a.score || (b.popularity ?? 0) - (a.popularity ?? 0)); + + const best = scored[0]; + if (best.score < threshold) return { best, accepted: false }; + + const runner = scored[1]; + if (!runner) return { best, accepted: true }; + + const sameBook = + normalizeText(coreTitle(best.title)) === normalizeText(coreTitle(runner.title)) && + normalizeAuthor(best.author ?? '') === normalizeAuthor(runner.author ?? ''); + if (sameBook) return { best, accepted: true }; + + return { best, accepted: best.score - runner.score >= margin }; +} diff --git a/src/connectors/queue.ts b/src/connectors/queue.ts new file mode 100644 index 0000000..dd14898 --- /dev/null +++ b/src/connectors/queue.ts @@ -0,0 +1,115 @@ +import { withTransaction, type DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import type { OutboundEvent } from './types.js'; + +/** + * Coalescing retry queue for connector fan-out. One pending row per + * (user, connector, document, kind): a newer event of the same kind replaces + * the older pending one, so bursts of progress syncs collapse to the latest. + */ + +const MAX_ATTEMPTS = 8; +// Exponential backoff in seconds, capped; index by attempt count. +const BACKOFF = [30, 120, 300, 900, 3600, 10800, 43200, 86400]; + +export interface QueueRow { + id: number; + user_id: number; + connector_id: string; + document: string; + kind: string; + payload: string; + attempts: number; + status: string; + next_try_at: number; + last_error: string | null; +} + +/** + * Enqueue an event. The queue `kind` column doubles as the coalescing key: a + * later event with the same (user, connector, document, kind) replaces the + * pending one. Progress uses the bare event kind (collapse bursts to latest); + * highlights pass a per-item coalesceKey so distinct highlights on the same book + * don't overwrite each other. The stored payload always carries the real + * OutboundEvent, whose `.kind` drives connector dispatch. + */ +export function enqueue( + db: DB, + userId: number, + connectorId: string, + ev: OutboundEvent, + coalesceKey: string = ev.kind, + now = nowSeconds() +): void { + db.prepare( + `INSERT INTO connector_queue + (user_id, connector_id, document, kind, payload, attempts, status, next_try_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 0, 'pending', ?, ?, ?) + ON CONFLICT(user_id, connector_id, document, kind) DO UPDATE SET + payload = excluded.payload, + attempts = 0, + status = 'pending', + next_try_at = excluded.next_try_at, + last_error = NULL, + updated_at = excluded.updated_at` + ).run(userId, connectorId, ev.document, coalesceKey, JSON.stringify(ev), now, now, now); +} + +export function claimReady(db: DB, limit: number, now = nowSeconds()): QueueRow[] { + return db + .prepare( + `SELECT id, user_id, connector_id, document, kind, payload, attempts, status, next_try_at, last_error + FROM connector_queue + WHERE status = 'pending' AND next_try_at <= ? + ORDER BY next_try_at + LIMIT ?` + ) + .all(now, limit) as unknown as QueueRow[]; +} + +export function markDone(db: DB, id: number, now = nowSeconds()): void { + db.prepare(`UPDATE connector_queue SET status = 'done', updated_at = ? WHERE id = ?`).run(now, id); +} + +/** Record a failed attempt: reschedule with backoff, or dead-letter past the cap. */ +export function markFailed( + db: DB, + row: QueueRow, + error: string, + retryable: boolean, + now = nowSeconds() +): void { + const attempts = row.attempts + 1; + if (!retryable || attempts >= MAX_ATTEMPTS) { + db.prepare( + `UPDATE connector_queue SET status = 'dead', attempts = ?, last_error = ?, updated_at = ? WHERE id = ?` + ).run(attempts, error.slice(0, 500), now, row.id); + return; + } + const backoff = BACKOFF[Math.min(attempts - 1, BACKOFF.length - 1)]; + db.prepare( + `UPDATE connector_queue SET attempts = ?, last_error = ?, next_try_at = ?, updated_at = ? WHERE id = ?` + ).run(attempts, error.slice(0, 500), now + backoff, now, row.id); +} + +/** Drop a queued row entirely (e.g. connector unlinked). */ +export function purgeConnector(db: DB, userId: number, connectorId: string): void { + db.prepare(`DELETE FROM connector_queue WHERE user_id = ? AND connector_id = ?`).run( + userId, + connectorId + ); +} + +export function queueDepth(db: DB, userId: number, connectorId: string): { pending: number; dead: number } { + const row = db + .prepare( + `SELECT + SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending, + SUM(CASE WHEN status = 'dead' THEN 1 ELSE 0 END) AS dead + FROM connector_queue WHERE user_id = ? AND connector_id = ?` + ) + .get(userId, connectorId) as { pending: number | null; dead: number | null }; + return { pending: row.pending ?? 0, dead: row.dead ?? 0 }; +} + +export const _internals = { MAX_ATTEMPTS, BACKOFF, withTransaction }; diff --git a/src/connectors/readwise.ts b/src/connectors/readwise.ts new file mode 100644 index 0000000..c7aa4e2 --- /dev/null +++ b/src/connectors/readwise.ts @@ -0,0 +1,154 @@ +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * Readwise connector (Tier 1). Official REST API, per-user access token. + * Carries highlights/notes only — NOT reading progress. Bidirectional: + * - fan-out: push CrossInk clippings via POST /api/v2/highlights/ + * - fan-in: pull highlights via GET /api/v2/export/ (incl. Kindle, which + * Readwise ingests for us — the "aggregator hop", see docs/design/sync-hub.md). + * + * LIVE-VERIFY GATE: the v2 field names below follow Readwise's documented API + * (readwise.io/api_deets) but should be reconfirmed at implementation. Endpoints + * are stable and public, so the risk is lower than Hardcover's beta API. + */ + +const BASE = 'https://readwise.io/api/v2'; + +interface ReadwiseCred extends Credential { + token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as ReadwiseCred).token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing readwise token'); + return t; +} + +function authHeaders(token: string): Record { + return { authorization: `Token ${token}`, 'content-type': 'application/json' }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + const res = await http(`${BASE}/auth/`, { method: 'GET', headers: authHeaders(token) }); + if (res.status === 204 || res.status === 200) return { ok: true }; + if (res.status === 401) return { ok: false, error: 'invalid token' }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +/** + * Readwise groups highlights by (title, author) rather than an external book id + * we look up ahead of time — when we push, we send title/author and Readwise + * creates/finds the book. So "matching" is trivial: any document with a title + * (or a parseable filename) is pushable. We store a synthetic match so the + * runner treats it as matched. + */ +async function match( + cred: Credential, + doc: DocumentMeta, + _http: HttpTransport +): Promise { + const title = doc.title ?? titleFromFilename(doc.filename); + if (!title) return null; + return { externalId: `title:${title}`, confidence: 1, queryUsed: title }; +} + +function titleFromFilename(filename: string | null): string | null { + if (!filename) return null; + const base = filename.replace(/\.[a-z0-9]+$/i, ''); + const parts = base.split(' - '); + return (parts[0] ?? base).trim() || null; +} + +async function push( + cred: Credential, + _m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + // Readwise only accepts highlight events; progress/finished are no-ops here. + if (ev.kind !== 'highlight' || !ev.highlight) { + return { ok: true }; + } + const token = tokenOf(cred); + const h = ev.highlight; + const highlight: Record = { + text: h.text, + title: h.title ?? undefined, + author: h.author ?? undefined, + source_type: 'crosspoint', + category: 'books', + note: h.note ?? undefined, + location: h.location ?? undefined, + location_type: 'order', + highlighted_at: h.highlightedAt ? new Date(h.highlightedAt * 1000).toISOString() : undefined, + }; + const res = await http(`${BASE}/highlights/`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ highlights: [highlight] }), + }); + if (res.status === 401) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + if (res.status === 200 || res.status === 201) return { ok: true }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; +} + +/** + * Fan-in: export highlights updated since a cursor. Returns raw Readwise books + * (each with a highlights[] array) plus the nextCursor for incremental pulls. + * The caller maps these into the canonical clippings store. Kept separate from + * the Connector interface (which is fan-out only for now) — wired when clipping + * fan-in lands. + */ +export async function exportHighlights( + cred: Credential, + http: HttpTransport, + updatedAfter?: number, + pageCursor?: string +): Promise<{ results: unknown[]; nextCursor: string | null; status: number }> { + const token = tokenOf(cred); + const params = new URLSearchParams(); + if (updatedAfter) params.set('updatedAfter', new Date(updatedAfter * 1000).toISOString()); + if (pageCursor) params.set('pageCursor', pageCursor); + const res = await http(`${BASE}/export/?${params.toString()}`, { + method: 'GET', + headers: authHeaders(token), + }); + if (res.status !== 200) return { results: [], nextCursor: null, status: res.status }; + const body = (await res.json()) as { results?: unknown[]; nextPageCursor?: string | null }; + return { + results: body.results ?? [], + nextCursor: body.nextPageCursor ?? null, + status: res.status, + }; +} + +export const readwiseConnector: Connector = { + id: 'readwise', + displayName: 'Readwise', + tier: 1, + capabilities: { read: true, write: true }, + carries: ['highlight'], + credentialKind: 'token', + experimental: false, + validate, + match, + push, +}; diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts new file mode 100644 index 0000000..d84bcff --- /dev/null +++ b/src/connectors/registry.ts @@ -0,0 +1,26 @@ +import type { Connector, HttpTransport } from './types.js'; +import { hardcoverConnector } from './hardcover.js'; +import { readwiseConnector } from './readwise.js'; + +/** All connectors known to this build. Tier 2/3 are added here as they land. */ +const CONNECTORS: Connector[] = [hardcoverConnector, readwiseConnector]; + +const byId = new Map(CONNECTORS.map((c) => [c.id, c])); + +export function listConnectors(): Connector[] { + return CONNECTORS; +} + +export function getConnector(id: string): Connector | undefined { + return byId.get(id); +} + +/** Default transport: the platform fetch, adapted to HttpTransport. */ +export const fetchTransport: HttpTransport = async (url, init) => { + const res = await fetch(url, init); + return { + status: res.status, + text: () => res.text(), + json: () => res.json(), + }; +}; diff --git a/src/connectors/runner.ts b/src/connectors/runner.ts new file mode 100644 index 0000000..d148ee3 --- /dev/null +++ b/src/connectors/runner.ts @@ -0,0 +1,139 @@ +import type { DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import { getConnector, fetchTransport } from './registry.js'; +import { + claimReady, + markDone, + markFailed, + type QueueRow, +} from './queue.js'; +import { + decryptCredential, + documentMeta, + getAccount, + getMatch, + saveMatch, + setAccountStatus, +} from './store.js'; +import type { HttpTransport, Match, OutboundEvent } from './types.js'; + +/** + * Resolve a connector match for a document, using the cached row when present. + * Manual matches are authoritative and never recomputed. Missing/auto rows are + * (re)computed via the connector's own search. Returns null when unmatched. + */ +export async function resolveMatch( + db: DB, + connectorId: string, + userId: number, + document: string, + http: HttpTransport +): Promise { + const cached = getMatch(db, userId, connectorId, document); + if (cached && cached.source === 'manual') { + return cached.external_id + ? { + externalId: cached.external_id, + externalEdition: cached.external_edition, + confidence: cached.confidence, + } + : null; + } + if (cached && cached.external_id) { + return { + externalId: cached.external_id, + externalEdition: cached.external_edition, + confidence: cached.confidence, + }; + } + + const connector = getConnector(connectorId); + const account = getAccount(db, userId, connectorId); + if (!connector || !account) return null; + const cred = decryptCredential(account); + const meta = documentMeta(db, userId, document); + const match = await connector.match(cred, meta, http); + saveMatch(db, userId, connectorId, document, match, match ? 'auto' : 'none'); + return match; +} + +/** Process a single queued event. Returns true if handled (done or dead). */ +export async function processRow(db: DB, row: QueueRow, http: HttpTransport): Promise { + const connector = getConnector(row.connector_id); + const account = getAccount(db, row.user_id, row.connector_id); + if (!connector || !account || !account.enabled) { + // Connector gone or disabled — drop permanently. + markFailed(db, row, 'connector unavailable or disabled', false); + return; + } + if (account.status === 'needs_reauth') { + markFailed(db, row, 'account needs reauth', false); + return; + } + + let match: Match | null; + try { + match = await resolveMatch(db, row.connector_id, row.user_id, row.document, http); + } catch (err) { + markFailed(db, row, `match failed: ${errStr(err)}`, true); + return; + } + if (!match) { + // Unmatched documents can't be pushed; drop this event (a later manual + // match + fresh sync will re-enqueue). Not an error state. + markFailed(db, row, 'no book match', false); + return; + } + + const ev = JSON.parse(row.payload) as OutboundEvent; + const cred = decryptCredential(account); + try { + const result = await connector.push(cred, match, ev, http); + if (result.ok) { + markDone(db, row.id); + return; + } + if (result.needsReauth) { + setAccountStatus(db, row.user_id, row.connector_id, 'needs_reauth', result.error); + } + markFailed(db, row, result.error, result.retryable); + } catch (err) { + markFailed(db, row, errStr(err), true); + } +} + +/** Drain up to `limit` ready events. Returns the number processed. */ +export async function drainQueue( + db: DB, + http: HttpTransport = fetchTransport, + limit = 20, + now = nowSeconds() +): Promise { + const rows = claimReady(db, limit, now); + for (const row of rows) { + await processRow(db, row, http); + } + return rows.length; +} + +function errStr(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Start a periodic drain loop; returns a stop function. */ +export function startQueueWorker(db: DB, intervalMs = 15_000): () => void { + let running = false; + const timer = setInterval(async () => { + if (running) return; // never overlap drains + running = true; + try { + await drainQueue(db); + } catch (err) { + console.error(JSON.stringify({ msg: 'queue drain error', error: errStr(err) })); + } finally { + running = false; + } + }, intervalMs); + if (typeof timer.unref === 'function') timer.unref(); + return () => clearInterval(timer); +} diff --git a/src/connectors/store.ts b/src/connectors/store.ts new file mode 100644 index 0000000..1855d94 --- /dev/null +++ b/src/connectors/store.ts @@ -0,0 +1,170 @@ +import type { DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import { decryptSecret, encryptSecret } from '../crypto/secrets.js'; +import type { Credential, DocumentMeta, Match } from './types.js'; + +export interface AccountRow { + user_id: number; + connector_id: string; + cred_enc: string; + account_label: string | null; + status: string; + enabled: number; + last_error: string | null; + created_at: number; + updated_at: number; +} + +export function upsertAccount( + db: DB, + userId: number, + connectorId: string, + cred: Credential, + accountLabel: string | null, + now = nowSeconds() +): void { + const enc = encryptSecret(JSON.stringify(cred)); + db.prepare( + `INSERT INTO connector_accounts + (user_id, connector_id, cred_enc, account_label, status, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, 'ok', 1, ?, ?) + ON CONFLICT(user_id, connector_id) DO UPDATE SET + cred_enc = excluded.cred_enc, + account_label = excluded.account_label, + status = 'ok', + enabled = 1, + last_error = NULL, + updated_at = excluded.updated_at` + ).run(userId, connectorId, enc, accountLabel, now, now); +} + +export function getAccount(db: DB, userId: number, connectorId: string): AccountRow | null { + return ( + (db + .prepare('SELECT * FROM connector_accounts WHERE user_id = ? AND connector_id = ?') + .get(userId, connectorId) as AccountRow | undefined) ?? null + ); +} + +export function listAccounts(db: DB, userId: number): AccountRow[] { + return db + .prepare('SELECT * FROM connector_accounts WHERE user_id = ?') + .all(userId) as unknown as AccountRow[]; +} + +export function decryptCredential(row: AccountRow): Credential { + return JSON.parse(decryptSecret(row.cred_enc)) as Credential; +} + +export function deleteAccount(db: DB, userId: number, connectorId: string): void { + db.prepare('DELETE FROM connector_accounts WHERE user_id = ? AND connector_id = ?').run( + userId, + connectorId + ); +} + +export function setAccountStatus( + db: DB, + userId: number, + connectorId: string, + status: string, + error: string | null, + now = nowSeconds() +): void { + db.prepare( + 'UPDATE connector_accounts SET status = ?, last_error = ?, updated_at = ? WHERE user_id = ? AND connector_id = ?' + ).run(status, error, now, userId, connectorId); +} + +/** All connector accounts that are linked, enabled, and healthy for fan-out. */ +export function activeConnectorIds(db: DB, userId: number): string[] { + return ( + db + .prepare( + `SELECT connector_id FROM connector_accounts WHERE user_id = ? AND enabled = 1 AND status != 'error'` + ) + .all(userId) as { connector_id: string }[] + ).map((r) => r.connector_id); +} + +export interface MatchRow { + user_id: number; + connector_id: string; + document: string; + external_id: string | null; + external_edition: string | null; + confidence: number; + source: string; + query_used: string | null; + updated_at: number; +} + +export function getMatch( + db: DB, + userId: number, + connectorId: string, + document: string +): MatchRow | null { + return ( + (db + .prepare( + 'SELECT * FROM connector_matches WHERE user_id = ? AND connector_id = ? AND document = ?' + ) + .get(userId, connectorId, document) as MatchRow | undefined) ?? null + ); +} + +export function saveMatch( + db: DB, + userId: number, + connectorId: string, + document: string, + match: Match | null, + source: 'auto' | 'manual' | 'none', + now = nowSeconds() +): void { + db.prepare( + `INSERT INTO connector_matches + (user_id, connector_id, document, external_id, external_edition, confidence, source, query_used, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(user_id, connector_id, document) DO UPDATE SET + external_id = excluded.external_id, + external_edition = excluded.external_edition, + confidence = excluded.confidence, + source = excluded.source, + query_used = excluded.query_used, + updated_at = excluded.updated_at` + ).run( + userId, + connectorId, + document, + match?.externalId ?? null, + match?.externalEdition ?? null, + match?.confidence ?? 0, + source, + match?.queryUsed ?? null, + now + ); +} + +export function listMatches(db: DB, userId: number, connectorId: string): MatchRow[] { + return db + .prepare( + 'SELECT * FROM connector_matches WHERE user_id = ? AND connector_id = ? ORDER BY updated_at DESC' + ) + .all(userId, connectorId) as unknown as MatchRow[]; +} + +export function documentMeta(db: DB, userId: number, document: string): DocumentMeta { + const row = db + .prepare('SELECT title, author, filename FROM documents WHERE user_id = ? AND document = ?') + .get(userId, document) as + | { title: string | null; author: string | null; filename: string | null } + | undefined; + return { + document, + title: row?.title ?? null, + author: row?.author ?? null, + filename: row?.filename ?? null, + }; +} diff --git a/src/connectors/types.ts b/src/connectors/types.ts new file mode 100644 index 0000000..82e54fe --- /dev/null +++ b/src/connectors/types.ts @@ -0,0 +1,99 @@ +/** + * Connector framework types. A connector adapts crosspoint-sync's canonical + * reading state to one external service. See docs/design/sync-hub.md. + */ + +export type Capability = { read: boolean; write: boolean }; + +/** What data types a connector carries (progress/shelves vs highlights). */ +export type DataKind = 'progress' | 'finished' | 'highlight'; + +export type CredentialKind = 'token' | 'oauth' | 'cookies'; + +/** Feasibility/trust tier from the design doc. */ +export type Tier = 1 | 2 | 3; + +/** Metadata we know about a document, used for server-side book matching. */ +export interface DocumentMeta { + document: string; + title: string | null; + author: string | null; + filename: string | null; +} + +export interface Match { + externalId: string; + externalEdition?: string | null; + confidence: number; // 0..1 + queryUsed?: string; +} + +/** A canonical reading event to fan out to a connector. */ +export interface OutboundEvent { + kind: DataKind; + document: string; + /** 0..1 reading fraction (progress/finished events). */ + percentage?: number; + /** unix seconds when this happened on the device/server. */ + timestamp: number; + /** For highlight events. */ + highlight?: { + text: string; + note?: string | null; + title?: string | null; + author?: string | null; + location?: number | null; + highlightedAt?: number | null; + }; +} + +/** Result of a push attempt; retryable=false means don't re-queue (permanent). */ +export type PushResult = + | { ok: true } + | { ok: false; retryable: boolean; error: string; needsReauth?: boolean }; + +export interface ValidateResult { + ok: boolean; + accountLabel?: string; + error?: string; +} + +/** + * Minimal HTTP transport injected into connectors so tests can supply a fake + * without real network access. Mirrors the subset of fetch we use. + */ +export interface HttpTransport { + (url: string, init: { + method: string; + headers?: Record; + body?: string; + }): Promise<{ + status: number; + text(): Promise; + json(): Promise; + }>; +} + +/** A parsed credential (shape is connector-specific; stored encrypted as JSON). */ +export type Credential = Record; + +export interface Connector { + id: string; + displayName: string; + tier: Tier; + capabilities: Capability; + /** Which data kinds this connector accepts on fan-out. */ + carries: DataKind[]; + credentialKind: CredentialKind; + /** Whether the connector is experimental (Tier 2/3 cookie-replay). */ + experimental: boolean; + + /** Validate a credential and return the account label if possible. */ + validate(cred: Credential, http: HttpTransport): Promise; + + /** Resolve a document to an external book id. Null = no confident match. */ + match(cred: Credential, doc: DocumentMeta, http: HttpTransport): Promise; + + /** Push one outbound event. Only called for write-capable connectors. */ + push(cred: Credential, match: Match, ev: OutboundEvent, http: HttpTransport): Promise; +} diff --git a/src/crypto/secrets.ts b/src/crypto/secrets.ts new file mode 100644 index 0000000..81deb4e --- /dev/null +++ b/src/crypto/secrets.ts @@ -0,0 +1,76 @@ +import crypto from 'node:crypto'; + +/** + * Symmetric encryption for connector credentials (third-party tokens and session + * cookie bundles) at rest. AES-256-GCM with a random per-record IV; the auth tag + * detects tampering. The key comes from TOKEN_ENC_KEY — if it's unset, all + * connectors are disabled rather than storing secrets in the clear. + * + * Wire format (base64 of): [1-byte version][12-byte iv][16-byte tag][ciphertext] + */ + +const VERSION = 1; +const IV_LEN = 12; +const TAG_LEN = 16; + +let cachedKey: Buffer | null | undefined; + +/** Resolves the 32-byte key from TOKEN_ENC_KEY (hex, base64, or raw >=32 chars), or null. */ +export function getEncryptionKey(env: NodeJS.ProcessEnv = process.env): Buffer | null { + if (cachedKey !== undefined) return cachedKey; + const raw = env.TOKEN_ENC_KEY; + if (!raw) { + cachedKey = null; + return null; + } + let key: Buffer | null = null; + if (/^[0-9a-fA-F]{64}$/.test(raw)) { + key = Buffer.from(raw, 'hex'); + } else { + const b64 = Buffer.from(raw, 'base64'); + if (b64.length === 32) { + key = b64; + } else if (Buffer.byteLength(raw) >= 32) { + // Derive a stable 32-byte key from an arbitrary passphrase. + key = crypto.createHash('sha256').update(raw).digest(); + } + } + if (!key || key.length !== 32) { + throw new Error('TOKEN_ENC_KEY must be 32 bytes (64 hex chars, base64, or a >=32 char passphrase)'); + } + cachedKey = key; + return cachedKey; +} + +/** Test seam: forget the cached key so a changed env is picked up. */ +export function resetEncryptionKeyCache(): void { + cachedKey = undefined; +} + +export function secretsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return getEncryptionKey(env) !== null; +} + +export function encryptSecret(plaintext: string, env: NodeJS.ProcessEnv = process.env): string { + const key = getEncryptionKey(env); + if (!key) throw new Error('TOKEN_ENC_KEY not configured; connector credentials cannot be stored'); + const iv = crypto.randomBytes(IV_LEN); + const cipher = crypto.createCipheriv('aes-256-gcm', key, iv); + const enc = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); + const tag = cipher.getAuthTag(); + return Buffer.concat([Buffer.from([VERSION]), iv, tag, enc]).toString('base64'); +} + +export function decryptSecret(stored: string, env: NodeJS.ProcessEnv = process.env): string { + const key = getEncryptionKey(env); + if (!key) throw new Error('TOKEN_ENC_KEY not configured; cannot decrypt connector credentials'); + const buf = Buffer.from(stored, 'base64'); + if (buf.length < 1 + IV_LEN + TAG_LEN) throw new Error('Malformed encrypted secret'); + if (buf[0] !== VERSION) throw new Error(`Unsupported secret version ${buf[0]}`); + const iv = buf.subarray(1, 1 + IV_LEN); + const tag = buf.subarray(1 + IV_LEN, 1 + IV_LEN + TAG_LEN); + const enc = buf.subarray(1 + IV_LEN + TAG_LEN); + const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(enc), decipher.final()]).toString('utf8'); +} diff --git a/src/index.ts b/src/index.ts index 00b5719..9d9af0c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,8 @@ import { serve } from '@hono/node-server'; import { createApp } from './app.js'; import { fromEnv } from './config.js'; import { migrate, openDatabase } from './db/db.js'; +import { secretsEnabled } from './crypto/secrets.js'; +import { startQueueWorker } from './connectors/runner.js'; const DATABASE_PATH = process.env.DATABASE_PATH ?? '/data/crosspoint.db'; const PORT = Number(process.env.PORT ?? 8080); @@ -11,12 +13,20 @@ migrate(db); const app = createApp(db, fromEnv()); +// Connector fan-out queue worker (only meaningful when encryption — hence +// connectors — is configured). +const connectorsEnabled = secretsEnabled(); +if (connectorsEnabled) { + startQueueWorker(db); +} + serve({ fetch: app.fetch, port: PORT }, (info) => { console.log( JSON.stringify({ msg: 'crosspoint-sync listening', port: info.port, db: DATABASE_PATH, + connectors: connectorsEnabled ? 'enabled' : 'disabled (no TOKEN_ENC_KEY)', }) ); }); diff --git a/src/routes/kosync.ts b/src/routes/kosync.ts index 8ac4290..30f7772 100644 --- a/src/routes/kosync.ts +++ b/src/routes/kosync.ts @@ -11,6 +11,7 @@ import { import { hashKey } from '../auth/password.js'; import { parsePosition } from '../models/position.js'; import { nowSeconds } from '../models/sync.js'; +import { fanOutProgress } from '../connectors/fanout.js'; const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; @@ -188,6 +189,7 @@ export function kosyncRoutes(db: DB, config: Config): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); diff --git a/src/routes/v1/clippings.ts b/src/routes/v1/clippings.ts index efd9f9b..d67e1ad 100644 --- a/src/routes/v1/clippings.ts +++ b/src/routes/v1/clippings.ts @@ -3,6 +3,8 @@ import { withTransaction, type DB } from '../../db/db.js'; import { kosyncError, type AppEnv } from '../../auth/middleware.js'; import { isValidDocument } from '../kosync.js'; import { isItemId, nowSeconds, parseListParams } from '../../models/sync.js'; +import { fanOutHighlight } from '../../connectors/fanout.js'; +import { documentMeta } from '../../connectors/store.js'; const MAX_BATCH = 50; const MAX_TEXT = 2048; // matches the firmware's My Clippings.txt export cap @@ -130,6 +132,8 @@ export function clippingRoutes(db: DB): Hono { type Op = () => void; const ops: Op[] = []; + type Highlight = { id: string; text: string; note: string | null; chapter: string; createdAt: number }; + const highlights: Highlight[] = []; for (const raw of items) { const o = raw as Record; if (!isItemId(o.id)) { @@ -177,10 +181,32 @@ export function clippingRoutes(db: DB): Hono { note, color, createdAt, now ) ); + highlights.push({ id, text, note, chapter, createdAt }); } withTransaction(db, () => { for (const op of ops) op(); }); + // Fan out highlights to connectors that carry them (e.g. Readwise). Best + // effort; the document's title/author (if synced) become the Readwise book. + if (highlights.length > 0) { + const meta = documentMeta(db, user.id, document); + for (const h of highlights) { + fanOutHighlight( + db, + user.id, + document, + h.id, + { + text: h.text, + note: h.note, + title: meta.title, + author: meta.author, + highlightedAt: h.createdAt > 0 ? h.createdAt : null, + }, + now + ); + } + } return c.json({ until: now, accepted: ops.length }); }); diff --git a/src/routes/v1/connectors.ts b/src/routes/v1/connectors.ts new file mode 100644 index 0000000..ab32603 --- /dev/null +++ b/src/routes/v1/connectors.ts @@ -0,0 +1,182 @@ +import { Hono } from 'hono'; +import { withTransaction, type DB } from '../../db/db.js'; +import { kosyncError, type AppEnv } from '../../auth/middleware.js'; +import { secretsEnabled } from '../../crypto/secrets.js'; +import { fetchTransport, getConnector, listConnectors } from '../../connectors/registry.js'; +import { purgeConnector, queueDepth } from '../../connectors/queue.js'; +import { resolveMatch } from '../../connectors/runner.js'; +import { + deleteAccount, + getAccount, + getMatch, + listMatches, + saveMatch, + upsertAccount, +} from '../../connectors/store.js'; +import { isValidDocument } from '../kosync.js'; +import type { HttpTransport } from '../../connectors/types.js'; + +/** + * Master-sync-hub connector management. Same x-auth headers as the rest of v1. + * Credential entry realistically happens from a browser (token paste / OAuth), + * but every endpoint works over curl too. + * + * `transport` is injectable so tests can validate/link connectors without real + * network calls. + */ +export function connectorRoutes(db: DB, transport: HttpTransport = fetchTransport): Hono { + const app = new Hono(); + + // List available connectors + this user's link status. + app.get('/connectors', (c) => { + const user = c.get('user'); + const enabled = secretsEnabled(); + return c.json({ + encryption: enabled ? 'enabled' : 'disabled', + connectors: listConnectors().map((conn) => { + const account = getAccount(db, user.id, conn.id); + return { + id: conn.id, + name: conn.displayName, + tier: conn.tier, + experimental: conn.experimental, + carries: conn.carries, + capabilities: conn.capabilities, + credential_kind: conn.credentialKind, + linked: !!account, + status: account?.status ?? null, + account: account?.account_label ?? null, + queue: account ? queueDepth(db, user.id, conn.id) : undefined, + }; + }), + }); + }); + + // Link (or re-link) a connector by validating and storing its credential. + app.put('/connectors/:id', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!secretsEnabled()) { + return c.json( + { code: 2003, message: 'Server has no TOKEN_ENC_KEY; connector storage disabled' }, + 403 + ); + } + let body: unknown; + try { + body = await c.req.json(); + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const cred = (body as Record | null)?.credential; + if (typeof cred !== 'object' || cred === null) { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const result = await conn.validate(cred as Record, transport); + if (!result.ok) { + return c.json({ code: 2003, message: result.error ?? 'Credential rejected' }, 400); + } + const user = c.get('user'); + upsertAccount(db, user.id, conn.id, cred as Record, result.accountLabel ?? null); + return c.json({ id: conn.id, linked: true, account: result.accountLabel ?? null }); + }); + + // Unlink and wipe queued work + matches. + app.delete('/connectors/:id', (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const user = c.get('user'); + withTransaction(db, () => { + deleteAccount(db, user.id, conn.id); + purgeConnector(db, user.id, conn.id); + db.prepare('DELETE FROM connector_matches WHERE user_id = ? AND connector_id = ?').run( + user.id, + conn.id + ); + }); + return c.json({ id: conn.id, linked: false }); + }); + + // List this user's book matches for a connector (for the review UI). + app.get('/connectors/:id/matches', (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const user = c.get('user'); + return c.json({ + connector: conn.id, + matches: listMatches(db, user.id, conn.id).map((m) => ({ + document: m.document, + external_id: m.external_id, + confidence: m.confidence, + source: m.source, + query_used: m.query_used, + updated_at: m.updated_at, + })), + }); + }); + + // Manually set/override a match (sticky — never auto-recomputed). + app.put('/connectors/:id/matches/:document', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const document = c.req.param('document'); + if (!isValidDocument(document)) { + return kosyncError(c, 403, 2004, "Field 'document' not provided."); + } + let body: unknown; + try { + body = await c.req.json(); + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const o = (body ?? {}) as Record; + const externalId = o.external_id; + const user = c.get('user'); + if (externalId === null) { + // Explicit "no match" override — stop trying to sync this document. + saveMatch(db, user.id, conn.id, document, null, 'manual'); + return c.json({ document, external_id: null, source: 'manual' }); + } + if (typeof externalId !== 'string' || externalId.length === 0 || externalId.length > 128) { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + saveMatch( + db, + user.id, + conn.id, + document, + { + externalId, + externalEdition: typeof o.external_edition === 'string' ? o.external_edition : null, + confidence: 1, + }, + 'manual' + ); + return c.json({ document, external_id: externalId, source: 'manual' }); + }); + + // Force (re)matching of a document now — useful for testing and the review UI. + app.post('/connectors/:id/rematch/:document', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const document = c.req.param('document'); + if (!isValidDocument(document)) { + return kosyncError(c, 403, 2004, "Field 'document' not provided."); + } + const user = c.get('user'); + if (!getAccount(db, user.id, conn.id)) { + return c.json({ code: 2003, message: 'Connector not linked' }, 400); + } + // Clear any auto/none row so resolveMatch recomputes (manual is preserved). + const existing = getMatch(db, user.id, conn.id, document); + if (existing && existing.source !== 'manual') { + db.prepare( + 'DELETE FROM connector_matches WHERE user_id = ? AND connector_id = ? AND document = ?' + ).run(user.id, conn.id, document); + } + const match = await resolveMatch(db, conn.id, user.id, document, transport); + return c.json({ document, match: match ?? null }); + }); + + return app; +} diff --git a/src/routes/v1/progress.ts b/src/routes/v1/progress.ts index ee95f34..73e14a7 100644 --- a/src/routes/v1/progress.ts +++ b/src/routes/v1/progress.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import type { DB } from '../../db/db.js'; import { kosyncError, type AppEnv } from '../../auth/middleware.js'; import { isValidDocument, parseProgressBody, upsertProgress } from '../kosync.js'; +import { fanOutProgress } from '../../connectors/fanout.js'; export function progressRoutes(db: DB): Hono { const app = new Hono(); @@ -19,6 +20,7 @@ export function progressRoutes(db: DB): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); diff --git a/test/connectors.test.ts b/test/connectors.test.ts new file mode 100644 index 0000000..b3e13cc --- /dev/null +++ b/test/connectors.test.ts @@ -0,0 +1,236 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { DOC, makeTestApp, registerUser } from './helpers.js'; +import { resetEncryptionKeyCache } from '../src/crypto/secrets.js'; +import type { HttpTransport } from '../src/connectors/types.js'; +import { drainQueue } from '../src/connectors/runner.js'; +import { claimReady } from '../src/connectors/queue.js'; + +// A programmable fake transport. Records requests; returns queued responses by +// URL substring match. +function fakeTransport() { + const calls: { url: string; method: string; body?: string }[] = []; + const handlers: { match: string; status: number; body: unknown }[] = []; + const t: HttpTransport = async (url, init) => { + calls.push({ url, method: init.method, body: init.body }); + // Match on URL or request body; most-recently-registered wins so specific + // GraphQL operations (Search, mutations) override a broad 'graphql' handler. + const h = [...handlers] + .reverse() + .find((x) => url.includes(x.match) || (init.body ?? '').includes(x.match)); + const status = h?.status ?? 200; + const body = h?.body ?? {}; + return { + status, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + json: async () => body, + }; + }; + return { + transport: t, + calls, + on(match: string, status: number, body: unknown) { + handlers.push({ match, status, body }); + }, + }; +} + +const KEY = { TOKEN_ENC_KEY: 'a'.repeat(64) }; + +beforeEach(() => { + Object.assign(process.env, KEY); + resetEncryptionKeyCache(); +}); +afterEach(() => { + delete process.env.TOKEN_ENC_KEY; + resetEncryptionKeyCache(); +}); + +describe('connector management API', () => { + it('lists connectors with encryption status and unlinked state', async () => { + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors', { headers }); + const body = await res.json(); + expect(body.encryption).toBe('enabled'); + const ids = body.connectors.map((c: { id: string }) => c.id).sort(); + expect(ids).toEqual(['hardcover', 'readwise']); + expect(body.connectors.every((c: { linked: boolean }) => !c.linked)).toBe(true); + }); + + it('rejects linking when TOKEN_ENC_KEY is unset', async () => { + delete process.env.TOKEN_ENC_KEY; + resetEncryptionKeyCache(); + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'x' } }), + }); + expect(res.status).toBe(403); + }); + + it('validates and links Hardcover, then reports linked', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const link = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc-token' } }), + }); + expect(link.status).toBe(200); + expect((await link.json()).account).toBe('julia'); + + const list = await (await app.request('/api/v1/connectors', { headers })).json(); + const hc = list.connectors.find((c: { id: string }) => c.id === 'hardcover'); + expect(hc.linked).toBe(true); + expect(hc.account).toBe('julia'); + }); + + it('rejects an invalid credential (validate fails)', async () => { + const fake = fakeTransport(); + fake.on('graphql', 401, {}); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'bad' } }), + }); + expect(res.status).toBe(400); + }); + + it('unlink wipes account, matches, and queue', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + const del = await app.request('/api/v1/connectors/hardcover', { method: 'DELETE', headers }); + expect(del.status).toBe(200); + const list = await (await app.request('/api/v1/connectors', { headers })).json(); + expect(list.connectors.find((c: { id: string }) => c.id === 'hardcover').linked).toBe(false); + }); +}); + +describe('fan-out on progress sync', () => { + it('enqueues a progress event for a linked write connector and pushes it', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + + // Link Hardcover. + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + // Provide metadata so matching can work, then sync progress. + await app.request('/api/v1/documents', { + method: 'PUT', + headers, + body: JSON.stringify({ + items: [{ document: DOC, title: 'Foundryside', author: 'Robert Jackson Bennett' }], + }), + }); + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.3, device_id: 'd1' }), + }); + + // A pending queue row should exist. + expect(claimReady(db, 10).length).toBeGreaterThan(0); + + // Now the drain: match (search) then push (mutation). + fake.on('Search', 200, { + data: { search: { results: [{ document: { id: 42, title: 'Foundryside', author_names: ['Robert Jackson Bennett'] } }] } }, + }); + // The push mutation returns success. + fake.on('insert_user_book', 200, { data: { insert_user_book: { id: 1 } } }); + + await drainQueue(db, fake.transport, 10); + + // Queue drained. + expect(claimReady(db, 10)).toHaveLength(0); + // A mutation call was made. + expect(fake.calls.some((c) => c.body?.includes('insert_user_book'))).toBe(true); + }); + + it('does not fan out when no connector is linked', async () => { + const fake = fakeTransport(); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.3, device_id: 'd1' }), + }); + expect(claimReady(db, 10)).toHaveLength(0); + }); + + it('manual match override is honored and sticky', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + const set = await app.request(`/api/v1/connectors/hardcover/matches/${DOC}`, { + method: 'PUT', + headers, + body: JSON.stringify({ external_id: '999' }), + }); + expect(set.status).toBe(200); + const list = await ( + await app.request('/api/v1/connectors/hardcover/matches', { headers }) + ).json(); + expect(list.matches[0]).toMatchObject({ document: DOC, external_id: '999', source: 'manual' }); + }); +}); + +describe('readwise highlight fan-out', () => { + it('pushes a clipping as a highlight', async () => { + const fake = fakeTransport(); + fake.on('/auth/', 204, {}); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/readwise', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'rw' } }), + }); + await app.request('/api/v1/documents', { + method: 'PUT', + headers, + body: JSON.stringify({ items: [{ document: DOC, title: 'Foundryside', author: 'RJB' }] }), + }); + await app.request(`/api/v1/clippings/${DOC}`, { + method: 'PUT', + headers, + body: JSON.stringify({ + items: [ + { id: 'c0ffee0011223344', spine: 1, text: 'a memorable line', created_at: 1752300000 }, + ], + }), + }); + expect(claimReady(db, 10).length).toBeGreaterThan(0); + + fake.on('/highlights/', 200, [{ id: 1 }]); + await drainQueue(db, fake.transport, 10); + expect(fake.calls.some((c) => c.url.includes('/highlights/') && c.method === 'POST')).toBe(true); + expect(claimReady(db, 10)).toHaveLength(0); + }); +}); diff --git a/test/helpers.ts b/test/helpers.ts index d804f86..e1dde7c 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import type { Hono } from 'hono'; -import { createApp } from '../src/app.js'; +import { createApp, type AppOptions } from '../src/app.js'; import type { Config } from '../src/config.js'; import { migrate, openDatabase, type DB } from '../src/db/db.js'; import type { AppEnv } from '../src/auth/middleware.js'; @@ -14,7 +14,10 @@ export interface TestServer { db: DB; } -export function makeTestApp(configOverrides: Partial = {}): TestServer { +export function makeTestApp( + configOverrides: Partial = {}, + opts: AppOptions = {} +): TestServer { const db = openDatabase(':memory:'); migrate(db); const config: Config = { @@ -22,7 +25,7 @@ export function makeTestApp(configOverrides: Partial = {}): TestServer { authRateLimitPerMinute: 0, // disabled in tests (limiter state is per-app anyway) ...configOverrides, }; - return { app: createApp(db, config), db }; + return { app: createApp(db, config, opts), db }; } let userCounter = 0; diff --git a/test/matching.test.ts b/test/matching.test.ts new file mode 100644 index 0000000..d8ed089 --- /dev/null +++ b/test/matching.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { + coreTitle, + decideMatch, + extractTitleAuthor, + normalizeAuthor, + scoreCandidate, +} from '../src/connectors/matching.js'; + +describe('matching helpers', () => { + it('strips subtitles and series suffixes from titles', () => { + expect(coreTitle('Foundryside: A Novel')).toBe('Foundryside'); + expect(coreTitle('Foundryside (The Founders Trilogy, Book 1)')).toBe('Foundryside'); + expect(coreTitle('Plain Title')).toBe('Plain Title'); + }); + + it('normalizes "Last, First" author form', () => { + expect(normalizeAuthor('Bennett, Robert Jackson')).toBe('robert jackson bennett'); + expect(normalizeAuthor('Robert Jackson Bennett')).toBe('robert jackson bennett'); + }); + + it('uses EPUB title/author as the primary signal', () => { + expect(extractTitleAuthor({ document: 'd', title: 'Foundryside', author: 'RJB', filename: 'x.epub' })) + .toEqual({ title: 'Foundryside', author: 'RJB' }); + }); + + it('falls back to a filename query only when title is missing, without guessing order', () => { + expect( + extractTitleAuthor({ + document: 'd', + title: null, + author: null, + filename: 'Foundryside - Robert Jackson Bennett.epub', + }) + ).toEqual({ title: 'Foundryside Robert Jackson Bennett', author: '' }); + expect(extractTitleAuthor({ document: 'd', title: null, author: null, filename: null })).toBeNull(); + }); + + it('scores exact title+author near 1 and mismatches low', () => { + const good = scoreCandidate('Foundryside', 'Robert Jackson Bennett', { + externalId: '1', + title: 'Foundryside', + author: 'Robert Jackson Bennett', + }); + expect(good).toBeGreaterThan(0.9); + const bad = scoreCandidate('Foundryside', 'Robert Jackson Bennett', { + externalId: '2', + title: 'Dune', + author: 'Frank Herbert', + }); + expect(bad).toBeLessThan(0.2); + }); + + it('auto-accepts a clear winner, rejects ambiguous title collisions', () => { + const clear = decideMatch('Foundryside', 'Robert Jackson Bennett', [ + { externalId: '1', title: 'Foundryside', author: 'Robert Jackson Bennett' }, + { externalId: '2', title: 'Dune', author: 'Frank Herbert' }, + ]); + expect(clear.accepted).toBe(true); + expect(clear.best?.externalId).toBe('1'); + + // Two different books literally titled "Circe" — needs author to split; with + // no author on the query, stays unaccepted. + const collision = decideMatch('Circe', '', [ + { externalId: '1', title: 'Circe', author: 'Madeline Miller' }, + { externalId: '2', title: 'Circe', author: 'Someone Else' }, + ]); + expect(collision.accepted).toBe(false); + }); + + it('accepts same-book different-edition ambiguity, taking the popular one', () => { + const d = decideMatch('Foundryside', 'Robert Jackson Bennett', [ + { externalId: 'ed1', title: 'Foundryside', author: 'Robert Jackson Bennett', popularity: 10 }, + { externalId: 'ed2', title: 'Foundryside', author: 'Robert Jackson Bennett', popularity: 500 }, + ]); + expect(d.accepted).toBe(true); + expect(d.best?.externalId).toBe('ed2'); + }); + + it('returns unaccepted for empty candidates', () => { + expect(decideMatch('X', 'Y', []).accepted).toBe(false); + }); +}); diff --git a/test/queue.test.ts b/test/queue.test.ts new file mode 100644 index 0000000..eccc9d1 --- /dev/null +++ b/test/queue.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import { migrate, openDatabase, type DB } from '../src/db/db.js'; +import { + claimReady, + enqueue, + markDone, + markFailed, + queueDepth, + _internals, +} from '../src/connectors/queue.js'; +import type { OutboundEvent } from '../src/connectors/types.js'; + +function freshDb(): DB { + const db = openDatabase(':memory:'); + migrate(db); + db.prepare('INSERT INTO users (id, username, key_hash, created_at) VALUES (1, ?, ?, 0)').run( + 'u', + 'h' + ); + return db; +} + +const ev = (over: Partial = {}): OutboundEvent => ({ + kind: 'progress', + document: 'a'.repeat(32), + percentage: 0.5, + timestamp: 1000, + ...over, +}); + +describe('connector queue', () => { + it('coalesces same (connector, document, kind) to the latest payload', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev({ percentage: 0.3 }), 'progress', 100); + enqueue(db, 1, 'hardcover', ev({ percentage: 0.6 }), 'progress', 101); + const ready = claimReady(db, 10, 200); + expect(ready).toHaveLength(1); + expect(JSON.parse(ready[0].payload).percentage).toBe(0.6); + }); + + it('keeps distinct highlight coalesce keys separate', () => { + const db = freshDb(); + enqueue(db, 1, 'readwise', ev({ kind: 'highlight' }), 'highlight:c1', 100); + enqueue(db, 1, 'readwise', ev({ kind: 'highlight' }), 'highlight:c2', 100); + expect(claimReady(db, 10, 200)).toHaveLength(2); + }); + + it('only returns rows whose next_try_at has passed', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 500); + expect(claimReady(db, 10, 400)).toHaveLength(0); + expect(claimReady(db, 10, 500)).toHaveLength(1); + }); + + it('markDone removes the row from the ready set', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 100); + const [row] = claimReady(db, 10, 200); + markDone(db, row.id, 200); + expect(claimReady(db, 10, 300)).toHaveLength(0); + expect(queueDepth(db, 1, 'hardcover')).toEqual({ pending: 0, dead: 0 }); + }); + + it('retryable failure backs off; non-retryable dead-letters immediately', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 100); + let [row] = claimReady(db, 10, 100); + markFailed(db, row, 'boom', true, 100); + // Not ready immediately after backoff window start + expect(claimReady(db, 10, 100)).toHaveLength(0); + expect(claimReady(db, 10, 100 + _internals.BACKOFF[0])).toHaveLength(1); + + enqueue(db, 1, 'readwise', ev(), 'highlight:x', 100); + [row] = claimReady(db, 10, 100).filter((r) => r.connector_id === 'readwise'); + markFailed(db, row, 'permanent', false, 100); + expect(queueDepth(db, 1, 'readwise')).toEqual({ pending: 0, dead: 1 }); + }); + + it('dead-letters after MAX_ATTEMPTS retryable failures', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 0); + let t = 0; + for (let i = 0; i < _internals.MAX_ATTEMPTS; i++) { + const ready = claimReady(db, 10, t); + expect(ready).toHaveLength(1); + markFailed(db, ready[0], 'again', true, t); + t += 1_000_000; // jump past any backoff + } + expect(queueDepth(db, 1, 'hardcover')).toEqual({ pending: 0, dead: 1 }); + }); +}); diff --git a/test/secrets.test.ts b/test/secrets.test.ts new file mode 100644 index 0000000..8ac49b0 --- /dev/null +++ b/test/secrets.test.ts @@ -0,0 +1,54 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { + decryptSecret, + encryptSecret, + getEncryptionKey, + resetEncryptionKeyCache, + secretsEnabled, +} from '../src/crypto/secrets.js'; + +const KEY_HEX = 'a'.repeat(64); + +afterEach(() => resetEncryptionKeyCache()); + +describe('secret vault', () => { + it('round-trips a token', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + const enc = encryptSecret('hardcover-token-xyz', env); + expect(enc).not.toContain('hardcover-token-xyz'); + expect(decryptSecret(enc, env)).toBe('hardcover-token-xyz'); + }); + + it('produces different ciphertext each call (random IV)', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + expect(encryptSecret('same', env)).not.toBe(encryptSecret('same', env)); + }); + + it('detects tampering via the auth tag', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + const enc = encryptSecret('secret', env); + const buf = Buffer.from(enc, 'base64'); + buf[buf.length - 1] ^= 0xff; // flip a ciphertext byte + expect(() => decryptSecret(buf.toString('base64'), env)).toThrow(); + }); + + it('accepts a base64 32-byte key and a passphrase', () => { + const b64 = { TOKEN_ENC_KEY: Buffer.alloc(32, 7).toString('base64') } as NodeJS.ProcessEnv; + expect(decryptSecret(encryptSecret('x', b64), b64)).toBe('x'); + resetEncryptionKeyCache(); + const phrase = { TOKEN_ENC_KEY: 'this is a long enough passphrase!!' } as NodeJS.ProcessEnv; + expect(decryptSecret(encryptSecret('y', phrase), phrase)).toBe('y'); + }); + + it('is disabled when the key is unset', () => { + const env = {} as NodeJS.ProcessEnv; + expect(secretsEnabled(env)).toBe(false); + expect(getEncryptionKey(env)).toBeNull(); + expect(() => encryptSecret('x', env)).toThrow(/TOKEN_ENC_KEY/); + }); + + it('rejects an obviously too-short key', () => { + const env = { TOKEN_ENC_KEY: 'short' } as NodeJS.ProcessEnv; + expect(() => getEncryptionKey(env)).toThrow(); + }); +}); From 02e167dbe0c158b2641cb0d8a9debb51d034b416 Mon Sep 17 00:00:00 2001 From: Justin Mitchell Date: Fri, 7 Aug 2026 02:34:40 -0400 Subject: [PATCH 2/4] Add web session authentication with signed cookies Implements stateless HMAC-signed session cookies for browser-based authentication alongside existing device header auth. The /api/v1 endpoints now accept either session cookies (for web UI) or x-auth headers (for firmware). Adds session creation/verification utilities and new auth/web routes. Includes static assets (favicon, logo) for the web interface. --- Dockerfile | 1 + assets/favicon.png | Bin 0 -> 1443 bytes assets/logo.png | Bin 0 -> 16645 bytes src/app.ts | 15 +- src/auth/middleware.ts | 24 ++++ src/auth/session.ts | 85 ++++++++++++ src/routes/auth.ts | 137 ++++++++++++++++++ src/routes/kosync.ts | 2 +- src/routes/web.ts | 307 +++++++++++++++++++++++++++++++++++++++++ test/account.test.ts | 115 +++++++++++++++ 10 files changed, 682 insertions(+), 4 deletions(-) create mode 100644 assets/favicon.png create mode 100644 assets/logo.png create mode 100644 src/auth/session.ts create mode 100644 src/routes/auth.ts create mode 100644 src/routes/web.ts create mode 100644 test/account.test.ts diff --git a/Dockerfile b/Dockerfile index 9284e4b..43c0ffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ WORKDIR /app COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/dist ./dist COPY migrations ./migrations +COPY assets ./assets COPY package.json ./ RUN mkdir -p /data && chown node:node /data USER node diff --git a/assets/favicon.png b/assets/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..42f6441da6a33e5751a763a46d477f7271bced05 GIT binary patch literal 1443 zcmV;U1zh@xP)Px#L}ge>W=%~1DgXcg2mk?xX#fNO00031000^Q000001E2u_0{{R30RRC20H6W@ z1ONa40RR91AfN*P1ONa40RR91AOHXW0IY^$^8f$^4M{{nR7efAR%=XLMHJq9?>?ZF z#e&MCY*$;Yl^ROL2b&-)5MpA*#F$1&w2_2h_<{XRO{8jLV~jQ=B&=bJHno~G^zlP$ zRt#*lgdbR~wuv_WFn}dm6SN8o`?~9Q&dj}g_l5j%c6aXFbLM>CnRCv}tn%`E1GfLO z+5ZzRaX zf&=)3&h9e|yU(4X5CJ-7%H_yIplX`K=}Z7xQPF;+F|D0ZsiEON%zqO%t+PJDRw z#?N!V{qn18Q?ciP9YkQsNSc76ps5JXwdu=S%POWWhF4c2=TDyg>6;(Q?z*GO>v1`q zc13k)c9+X#x7)AJUj1z3lOJcUB;vG0wDRhT?REEO5vW3k9`3Vq4u?6D6bf7~i_Zz*%%zWn>2=Z>8if9Lqkxkb|)163mzMRk~RnFGoseF>eD z$>~*;=H_N_1cSjh4!{2HE3b9FxF6LrG5%@fpA{1#JzxML;Ib06YY=c)C5@$6VC--> z`uh4hJ3EiOdE|I7n9wtFm_{eRgqNsXMbQ=7hgcSV6n#-`)T65DIOtbsPCd=<T2~-mrI|wLOG``h^YaCP)>AMrIVTy!3ancXWn-UPnD_a7!^6YT zXmmZa4d?^`M^=QF&M>lDTwLt$?{9B!4~N6+<6u{JcXxX|+Y&gn$k>1|%arWJ>6nwl z>FMc?j*i~m-i3t)%*ss+78(kL#>U2OD=862DKN-G+NlIgE1)qE^G}FLB^&%29UX0K zYzzbfu~^Iy+bSz7hlYkGCnww5+HkbsxR$6uFc2}Bcd|t0^Bs3n4hQBTV%X1!2F7l; zyQ`~fU|_)O^_qOhDH`1n0$qB{1X3m*bTOG$*#nBQ%~o4mJ2*I4RaKRxW&neMs7P;A z5#AwTE&Rx`6tC_rrThC1Jlp%i_t&o0@2N+PStCYsz#u>*4$0(UM+!OvbtDSYNMmW( z8GBoO9nU{qSwmjm+pzn>M`uuDAD=q)*3qMWzhBcd$YDn}v-lbyl7Q4uDBNe>tJU{; zTb_Gz=ic2|*wWvBVP;Wz$ukFEs%?5`;^g_Z*0!FWo*Kbm zv;-(k$d;l?K*Jy;I|W(f+xn#jE^rbFE{LKX7r30 zqzPaS=J;ByuH}(9$}wfDZ}*jNt-z%pU0tIc!04b6F^t9&I6EKP=ik0(*X7VO?%D^N z8emTDTbh#l*OC6VxE#2cxGHdW$R^zv#zF@!#FZPCAK%E|7MSDKj;`u-pr&sjK5n5C zArfmK4$c{#9Cv!pX&E6(YhjX1;@Tq)-8jrS2|R~#$nfa6Wrihd?WJ&gUUD#ZUc%8 zAU;9g44W`6qaukAL^fp$OMC(m0ucgX4J45CSMS%~IaT$qb8mNU-@f;DcjzJ{cloya zo%*W&^{+a0MxFTe)53_1NUNn7>=N4fX{LhE$5uqYunUo03XUC<3#nPNhaYFF5$k?Cbb~iprWX zhoa!Jq(E}qo>S;aDZZt6+>Kl|ev1{ISNgk9oiVjwZ>7xHQ6RHnNrl9E>E}^R)1D!4 zbqd=oy&%x0*ML@S>=sM3NWGT$EK>wANj;x4%NHTNu69uy(_@FCIA|g%oxXuF_X4&$;PEvq9gi`eXc4JKcM8xlCGel1^=4lO;k5|DO#nTuJWVl zEqQH8&NcZ*=E@6`nZu})8<%V>idBtOF#ko#3dzS&eNc6JKvAVvraJhgstl7IzH*a2 zdsXI#qsd8|v8i>XI>|&uB`jGE0`@MM!L&`%(~ZS|Z4`<{(c@6fhoWq0jCHVO6LC+d zN%*b;P2xS%u(QIZq-=`PdQg19B`1tkg$5xwBO1*XSNbH@kKsp%aUK3PY`7|xfnw1e zRg5^q$r=HaG7C~68~t13x7?t`f2uu8=AtrAMMOmrGa(Nls65wiQ903CN(xm7jzbly zY#RwIi~IEI31qR2A^VuUOJto4SvSu{P-Nr^7{pbxJ~ zVni0DOwpq41MEhSYm={wqVPb_U;XfrVPdfa?y}ky7n9jX3T|{P*epvRXW<;78GRhr zu$)QxI0Aiz?&LO*BSbcfg;+%rD`f6k9VLm+wga&mU>dfjYq(e}4kVx#XmK-Bl(sP) z6aI8~R#2@U^K0zJ2lB+^waJo4|IdmM7 zie)5Ze48_#!jfA|M($At&fTa1q1Qn(ENd8)ND`$?aa?7milC&thLCoFafnsXnH@um z(Q1kaV=&0>+8`!PRE3=@g_TB|k&+T5VdaoRU_%UJd8-s$m4Q5-aC0f!Q9&1=B1J6D zBWAE#H&;7Ig3t^?IejJ7szT04*)@gWd9nlo&633clPA!o>u0++h0UCoffTSG&!5?~FOB1TRi4NM3Y16pV=o(Htb z`%YE^fF6tp&IylxwOL6Lp#?pvd8o>Gl&s(PsKdxXr8=J9Y4z3(5T_y*|mf3~q?*OP~(e&VI*kCIb8YO+w z6DO%&f(bkzg4`7MnN7)^lSSw*_A}LCSFoU4)}uEobE2wW3vS@8`gJ(9MhHe<)HzJ_ z#TL7YqA=Ur>N$&Va|I&!?IF16w!<=bibbM36i~t3_wvdiTY~R_EZ2yS;r<%86hAD? zS=8b2>zo^sk`m_wLoQYbZG#U6lFejMl!C;-6wD6Vu^A9I!T;Ud}Iw;)FxSip{yXpDPS{6=wuclsu5cW9($+58L=wG?Cj2$ z8%4ZSbwE*}IqDMtBLm`rf#bCYOwtQVHRxH*9<$Q2!Kx1N@*GnafHvu@431(Em??M) zg9T-Zu84ahw8)RR7$7c~B<>lsloh37#veskd=3pQI)ySx3#gNV!a`_^WmW5MP*k<(iCmrM?^>k&|p6 zOOarLR?#Px8ZxH~WvzAI*v-Dt5?XF_17()KJA~EsP)n(j9T9M2btql(_WD zAb}wQtrE0M1(Yloq7nsp3rNn)fHKGktED2xgC!k`?ot3##EXfM*C$1P4!}A;H7JXD z)i@Ej=l~`W4ud}&_Qm`{2ptX)+i$T1g*c!Ku$4u(WMP)ME)iN%BI z0*T}?emDSs6~xvi zEh!Qki~*_SH-p_1Proo5;>^R?_DBa@VS*usQq!;HfG&u%*1%j%8W$)`Axz5V!(|4;ex#+JT@6eK!e9l@P8d5N@g33@0Pxctf$6T5 zC64ntf@z++GmhVK)I%re)3z&?4!)&=HOGez$u_iTikyWDRxqOnCl87Z(?nlM46s8U zacA}@vxIUym`YC!^Tq%ma_(C6QDB(C2=OCxZnT3|y{nCuBPtuhSd%4BhvuyeWRX4# zdK!Ux@XV$-TS05vw5x53g)l}DkD?N=%upvliTR_NL DkD>6baiSb}1VO zp+)ENBwIBD8L=(yE8|4ot0LtS%_;#*42Yais}Q3B?2`;2pfw|P$=#}+ZBl(Ii=nEe z^(k`qskDKge(Z^rkF0p+iKkv&`^wZUH{W#YXy5g5sGhno6~! znX%Eirqy^CXtFXmBFKy#k!|!WX`CT;Hb91lAs6E{6{;PO8~53yAc;VUB1A{^#Vc#p z-gCok58QOe^G`pU-LVN%ChjEtRf)-Z{6G$0LG@P4GhTGtn&~D!a=?Klt%o3vb@wA=kY4(vrouzx@2_ z&9~iZ?CZv%=L+O#EONwjI;GR9tnrn4YE?(_MWR`7fFYnFXaq<_k@hHroy-E*5Dd?O z4ffPM`fhswR{rVnML+!U?LWJrd;R*p?^yTx?mygi&zNyzx7uL_j*Ghw!DkTb!pH$g zD-}!AWhs~JeHS4BuWZ%bi*9N3&>U7ivf}PVw~U`WaoTp%1*l0?sVafqKJs}R(Yv%= zAt}j1kv>_jkMMPIx$fkS14I%i+qzwTHo(6B;_7VvCjTFHw!*0B}q6?2}Lb>ibuh z-DSTH&dkg1Ub=kgLlZZhGHLTEaC@}6&xSCd^G)9Z4AX?P#S?1q$nJcQ5ibbhkjw@~ z{U)cR1=6=sy5lk70|BK@;PSxlG4>$xIB>+UViSyAD?^ao!8yG zcJ<56-K*~P>o=J+vBx30fH~c-g4Bx>S;C{aQ{~bc#UdPlfP+Eq+v?BrD_7lc`OgP< z#-|^9a_Oyi7hPSOP2ZyE>XJkah0M9+0M9OlAd9TG6z2FuoD^KffCw!N17-_m?0`dc z_s9nm9LT#u3^-m{v*wl`FZ%ffmp%Vf9{2TE*1q)8thxKRVlroRYcObwG^=HFgFfTH zDnKF+ywQq>m$!tJTm8)Q*Z<$2EV<#f!$1Dln{G8t;DVT$fNqE|DdSomrv$gx zk(B!}=BFUf0>vLIiJf%Xlzm~nH_Fp8Bo2*2BYNnL`+sra<$cF}jmuNZpLpe^HRHy= zj$Q0$+36w_97;<=wTLfmg~bA0N^nGhmU|-8{}12!z={VSse7bx;*(A~Y2CVYja&nle9!f_-FxHj4*8qE+Wo+N zLjW$4VA_}<3ga+y!Zw&g)Pp(OTX`vv-iVE56Eu?1ORszN)tj!m_V>TM zu^o)%%P+oUem4CMj*Td)`Y&s46EshxKet&KXgwC!yv)l;?8hWC)`zj3Jb7|_as2Ve zFI%?kV;}oiS65eqS5kJD*M9G+|2*sbNAG{o*eMM1!5<~qZcvMg9#dNs6eA9W`aSQv zDtfB0t&8fro{e?OYJ(yj3XiRKfDmiyM?G=heK*~4>FNLWyPq#^6A#X1nLj?{3cU2+ z^cbey26WN^DHbq34jVfwB`>nf%LZlo^y%lFciz4C-n;wmyElGGkN>|vUGbF*e|G+V zu6=n886jbptC1k3$?-*{AW0Vr{S#HKD7s8ZqK3(~)gxb2jnWD^YEe*E&qwiP)XM_B zi;HFodg-|rFaMVde*W+O`OKf6>c@TWuUMg?3qSNrEeBB|#;Qn~^@;;B1sXergGag%NaWY5-G{8DOL!0NkKUS4zCPp|*|FMqTC)rQ>t03{VYvk+L& zEHj}N$W2?d?*M8anzdM>+*-k)VZEl$*ui!ocN;fZBIC!8Kl$X7@4ox)V~;(y@x5Mr z=J`dJ{?`vaf6g;2Ru+c3qdiHU%l? ze4c!NWb)+6mt1nm!i5X>-+%wc_j>xVColf&*M52NRjZ$UK0+l*w=v-=5^R(l7@0A4po_zSRtG;&eHQ)F@udZngo6wNda=i6G2`W|mpgBVj zrT>Z*D?*{QSdbxOXX?ZO(pdeKW?dlzZNp5I19k^sK%JmqvUzDLX6*j&3orZrf^+_K-yaLp zjHD%|WMH?tKLIsCmf{s#1nRHMJT!?k?}W@%I;g_Uv1Jx~P5iPko8D$5n>U zf18ktCAb(eR!vHO6icBJ5uTUOP^B$x1E@nOLKwL3PSRY=xP0luKltLeuKw1gudG?4 z(Vr@l=@8Y}Ez!R&fKaMlB;X1DqyxV6+zXd{;ak^y`_fe_n#_&1M3BcLt0saLn=&d= ziU?iu>(p&*So#wc;tUodXwpwPr_C_iDWR=YT4~dmS|QEFj7x63{gL}0ocrEG_n!CG zG2_MpkO11#OFuTMYzWHdG!%_%QM zE^;g=3nUHld&5>)uRr{5$yZPM+>O^*WUP*lm6-U>wXjMO;-;{7m!pSxg{!;BEF)auF5Q@jl0)%|M&N= z{Pt)6<$>Gp8B*;)3CUlf)$Fy_su~F|rD>AvMSItvH0}z6=Qmx+Z~Tb0v_zXVVDSk~ zdvB*S7c*XU-X-V#(^nq6XDK5ao6ZhS5!!6(Z)E92J=9e)sH>AXeRn`0@ z=>?bUSmyhJ&7pZ(L=k&Ax;~N*PeC*Hd{7ub^K^Mv75Trq{sL4E!SSTVa z(f7HrKBWQ0A_)d@dI?k^i}G7L9s5CQrG%2`h+ZX$G2^-Cp1bX~+cv(}9XH(aucw@T z<4>7`(FH&J#y_07^07aYu%nh_ ztmMAi?mY7&pI&(7&(;lzxUVlE@Fox%LJ31K-2%Am{EJM8YIOJ-u~-(^k@;N?zKk`o z%nVH_*pn2>V09eWHWxGg`H{!J{PBOf?AzaaVb!y#il*{_%m29Sf-ijI!Y}>X+Lza~ z^Qv1Xx^QPTw}r857KfU=2)qU-8NpBE6D~HaSUGC))lL2furVJxB}OAxonO^ScL>r% z%vhG?Z+^b$jN?A}+h1N^6h-&C^;duQKfd~@1xs$dV@Sn)8xRV(;)ae|-p3}?%TEJp zmK3%ByccPJdC!kfJFmkUx#UR7F{p#rbR6XPb+aS35MJy15&Lv37`wn%kKsaSkuyt+B?2k-mCaMddDWdR=bTC>z^`3Z!L!9x8SsPCmAx6hs2v^xp8T0cv&NOuy(1W0aK`k2Z&VwlxtMWC3+b{?c|sFCgQg_*>dalifc8|({dm1Khv ztcaloVCg{B5?Yxb>titMDVD_p)w{sdNC13tX?ps}mR_pK>+@05%mKYrS@X@h+g zEfFRQwy>iTgTDk9rx?x)w`qU=Wr-E>NHs+k!tzeIz{--^`v{jmu~a zxsdR+B6-UUXlB8>c%9IVS`<$K92V{gi)-|Pn(HW*m=?wivRs}EL;+2uEP<|BVinY= z_`}L#ZDc~mMG;|d64ES9mT1v_I?nHK-hrt;gVbwOn2{OgYt6YPlj^2a)SN?&+=x~U zo`4Gq!3s#V1h6ifrQu_IS4$_6ZB}cvY7{HW-N3vKiwBhZSB-{_WO#{MK+UzJ#fWrD ztjfybuWXBK53Y;LWwhgA7>Lw5CaXSpC#_QmIlzKhgIa@SFf08IcHNf<)^W-4--vWX zxP}#y7kSYMxx_HLhRU`KLSY3H6zZmdbH%a~5<~|&!HW!)>)FYzG-MZO@Jv@^Z(&?Z2R!wdR7!Sl7Shw{#30Zm=th!(MoBWh9&NiXo&vNf(#XAXNr)I zCzB9H0M_7$5|Fk$wk?}TfSa@6-Kv)BklV~z7e|x=Hsd<}k_{edRL3=nN<^?&a^1tQ za9|?iE9mhUTKxt3d4jY6 zBchTpF{OIac}7fD=fy07vYd*rDw`wNHya#Rm9ER8sM47MqkOImxI&}E?6|>cqVuO# zVJOT;vRGIp4bg{uFpv(nBhsWGs~APmil$3g7*7}XrME_f3EGxcfMQ{?VFin&0UaA( zP-72eQ*Bz3K4*{j^trImBF*|Au~(g!KFLGY=GJL7DJX7F1C}R_*$N^oV+gTg$SoGx zOl2de7+2{ht(x|!Ws+`|j@5PNBFC+b$e@UHbFqo5l=0R!h~)d*k{*EkPn zqAS#FaYkl@0Ofv$n3fB-jdDk!Me>xh2u! z?FE-5gmx_8+YC=(=Qi`i`DG>FNxBu+Tz9&7xjes6ENnFt=R&^*gxms!BrYigsIr2% zFDi=6b!Ns?3n{MGT~qvsBVJqGKv|Yswc3L`)OTc6ZEe$fD-<$%+Y#xsK(DC)D~mYO z;PWcTcLiYMNl6E^gJ2d@4-W)%b<-{!APZP)(}XGYGeL5P>B!#^>68%KshBp28Ir8@ z*Lr#Su76ljDmIF>;*SFYAszBUyc4uv1`^8H3%pJvxP;_#*vzgV$dPCREZvB-RbY;S zmN8fYSJV#p(deMO3ls~IRculHX);B|W0L42tYv`Z@>&`i_bvzkT%O^Exo2{SRxYG7 zBK;H>av2we48SNPbtoIY5i~~cb24AaMqw9THtUK?#bIzPO3l=O`wQALdZr6x3D%y0IYu1%e z&0txn(8c^>Aq2#84rI8-hq+uKerk3~95AecSOp9d+M*_x1ZOjz}iDVjShMd#r+&ro)VI zOGYjz?5}@tI3grxjCq%}FLZ6*5G-K?E#X2v!^LQhi0U_fR!r8gTyez}vuDpf^UO0F z-(^JVh(v!nzcj-;l|Ly>0|`p*3MI5G^3da2r0 zFN$Oj)#w%KR>{GB4EWGw;>EKqgONczO+YnAHQaaKeMcR2)IkRwG!ip5FOYPJmM67k z3|bXs-cP%orF{}9pXdCPN>}&fQaY@;VOtp3tB`3ng?cS+h>WyN#?$|#mJsgOUw{2b z%-Dpmt=A^AOr!dg!6)3()>`IJi)S4Bkjl2m5H*am@>m+gk+m@b4{B6Z?W9=KD$2^i z+|OfotWeFBxam3W5zk>oO8$_hxEFI572F_ z#osA5ht@Aar&Vxv6!I+}I9LH`FqmEh%n^?Y(kWA>G_X0`d@u?I zq_IR3rU$AIOc+ZQ4%jF+?=P!vNEPaAvs)F>6A-cSBIgjS|YK^}YRf~~hG zA(m5;9d=iw4o6nzqJ93*WFw9lySlngJn_V3%a)yR!U_G@O-QE99%SH&IsYPf4nhuN zGgmF(*BU@MI`UZL`35)tHFkCY8;R3xu#%92?|2zB#3<`nu=dHu7cu1)L?g`q4Uih*7?g)*e9nHr@hT$z@O&42{6ja(Ze z%@_wIBEvmqoIij5J@?$xK-|}>Fiu#|K%SgC zGfUR%LHrCI<+Omyz_g0*L5fVKwl~IO1+Bg?CF6{9D><`FR`Sq7dbFQ19An10bLTEx zxbV_TFP%Jj@&FG-*yxJ6mQxhyY+sR2@e+oUMl-Qau<=IC&x8hNm+*MKx;!(3KvPLA zSS`RZnv`0U(P>hKQOr1c^yrH&x@hs@#q;LPYt0cQUgI(~PG$}0bQEDOapH~om|e`i zGh>Y(0>#SP36i3`nq;I;A96^*Jk(DefGh4S!m1}jJZ9|b>N@G9la?)8_K}Z#q-6(% zniaoXDrlD2lr)go7es88Gc)9;h~}*ue19xFL9n+WQxSQ|l$9D|{6KE~OtSQ63vSfF zWoX8XG4A`$cfQkH+}E3EVW(msLcm1@>XfBnf`pPlz1!$9J2O<%Ejw#&OQ1RdRgCiL z!{8Z*baX4|VyH$mo{Va`&bgHCP;!I*S6PN+9o z_Xes{w300=N5PkilGk+Acx7Ta7+TMLZpf$-rVtvPrqfv&teCNdao>i7Ywo?kE_uS^W#^1eE~sRaS)4Hsa?Nw^By81^tF4iXGTJYKwF@W^8@j z*RW_1<@5okP5|B!5{YpC;8}sX6~nbl_X4(xWmSfml-19=Swb@-P2FgiNbbjUBwN_L zbRcGIOWfDDFz1sPtXYckKxku1enZ5c!tHY5uc zEGUXzR{8J0%$PCbGoSg)@y8$EQG{LzCPJbC(i|`{kI?x&V8YO9`NVc0EK9`>H8Zp7 zC{{=Z1Um18>K8uurvX>fVhA^ieYP@zOv zv7opub(kC;ZmROO(IOC7jsG9!1jvFlV@^L+ym#wrqa@@~cEQ-qGLe*h*oxjQmY6ypDx@<=(R47YW`#WrY^_w2 zn(Z1+bVGwOVZwx+ciy>G+(gR-BRiLEgIA~`?5YZoMOC(lZ0cAq(iZ_JOsNKy3^NUF zZUxLVWL$unfvL9uw)d*);VY|GuU@cV!Crgqb;Aue4ED9u5pX`f^fDOo)x`tZtO;e4 zcC62%jAI`|*_UETfNFiWEE>csgX%&8T%~CsYcR1fnk$t@AAR(|0}uS*2S0fK{r3;{ zwK(B4(>-DsD?331%&=s))}f>Phd4#Th;$uQ%RG2IEf6f8#Phsj`OZ z_s-gg#yi@|J ztH~+rjvAPL2ZDVIR7KyKzCXi9&N$` zxFjjg4=A@4fea-~%5x z_~3&dd+f1xURQ&{xI&g2KK~X>oKl3-n;ZdH`v9b3kfL^0Cmuqa*lnK z*eC|d4wF{89+tAxteNjV=6y5woV9*^_wN_q{EN%}>xtzn8r*l$qD6D&%sJ(hQ%*kl z~>k3oc;vRVoY>2MwLBfxE+wX9aZ$pwI+hp14e8FLt1WeHyy5!Ih#Y-gel$ z1s8w)f4t)Wu1ei{{CU~)SJ&vS_a675Pkrt4v)??Y@%0cMee}@-4mjXLANtS(4?NK3 zYp)e$72VBG1Cr|g%Ha9*s_Z?lV+aiwJ~dp(#sJQ-zM|$6HBM%t9NroRnzhw-+x+9V z&-%dM{msNpCzl*msw_iUR(Q4Sx##7Kowh&WbD#Lg|2}oxCa-J!$d_Gq*>1b-*7=w* zg@l>~w4u>zcOUNuHzgU%0ULal8d5B_34}Jw3xT|I&80M-Da)M3<-SR^!I#OKP5rBn z9`pBSe{S>XTPA&C?k=l`x9qW7@spA(CF63|etVvN(b)&T|6QZUjBflaF=o8#s;k=h zQ%s~5)R0qImM-{WEf?mHVJI9jSgd)2(wM~}j4dGp$e|{Ym}dD%3Y-ZoT|*}uG}-Hb zw|x4XFTDNT2ag&%rg}J1aY<#X#2X69Td7yipRwaNdDpS;Kkb6AykWPU8$Z{*_ul*d z_rL#;Lk@ZT@yFZwn?$^QvJcq`mpRiaYXMKVd4r?aXH@7fD-~Lhg%*LJl%ZOMHFn^Y z>cnc2E#?C|=@6F++9r|-V$ z_Jvpc?5W3}Xz;uX7cSg$&pl5)_0*4l{Nrtm85w{~Y>q`#0mhI4cGv}}ZCSOxFQz@0 zeoPjikf-^aAtK7cLA*rv)3k?<6kT2K`@3UL`qF21+~bYWt4BLj{eUvU_A;bT)Jj!$ zt4x2VuSyzK?79D&PyO2I`ycTR>QBsg#u;bKnKS3c8*gmakEC3A*s7FW=!Y1ZjJZnX zcibPd&=1AZc|R_lgMoxz2GIp zlhQt}en}YtSAty0QFe{)dgrn4``DNNappdI^eblEfB*g46*Gz-bgTh@=Afw>{>wCmsF2P8|EXagAS9j2XZ5r7yMi$1+qw#SgT1Cw|Tt z1maY9C?IZJyZSkAj@1ldX*=co7DMkFV{oKp7({fA93iY{=|%@pMLrtd+gD|n33SY z*Ba3H41uKp=pqITEL?x!RVDqOkjd=n4F{+63t%CvK8x%5`%cR4bN4y%?9=xBi@7oG zV*^ZV9uU1D*eWnxB^oXGG8rwYX6$)0cKqw+9sQT5ocsA1Gk0wKI+rY2(!!XLp#(t9 z0(N}CS%(}D#56yR+u|v9iHyZgI?rD_wGM zj!#h7$L>5vlXdO@ItyR(0b=cn?HUEs0x1GHc1bwA5 zFk+3^FjM5&EB6N|DnToO>be(uvnOCbZro$@imt9%Z{Pd4FMoQ!!w>9q%*Yy8SuDs1 z=hIl(Lc2BfcKw!SdlSSts0^h#Nr`}qX9o+@#~p<4R1|NT_m&gCaoX+&?w7(jQHbtL zb)*ugv`~*hML5Cv__qrc0n3*bJeEO%06YS7*Ql-ok3M|PR*2~#JD zuCs!14JFE2~x}7}A9m-_D#>Tv?feJ%Rnd@lFQNrblZF85Trg8%K18)|Ir) zso{;FaC@N){v_=b> z2-|fg-3le%4gIN(cz|5K<1iBT+3l@!cG+k4O;`Ww_rJWc?|BL;F=qVcH@`V)(xhd} zmi4yTCKJcoHAw-m{9LZL&}s{!_W>)GCZ}13B2qhL8Ei#iwngqJOrEysz)zY^UC{3P z@B5bb96VvlM0GS~h$#3TJ?f~F1tkVekmgS=#lfXp2t*MHk=G$0!EnIx-iARG8t6V@ zk(s$`%;^1(I&7DHcfaMwi( zz0-CT+~kl=gpfQgJZm96Gp*39u6^D)ci=~flDZnjmVY5@LQm>JMvob@_T@EC{;Bz7 z#e}Jo-g?BL^X9*2!jwsP_N-B*`-BW{g^*bs@p25UC@6&57AEXJ#4Z~YaD3cO5uAb1 zVn1aZDy4K-dFqy%?QzhXiNeZ1tr$pGh${sc&&;>H>Aj~MJ7eY>imt9&N+wq?@rD$j8x;BPaK|8B zCbA`7K$~i&8)4Te^Qh1`N>(XVN1It|wcm;gD@7bVX4H(`cAE9}y`NmR{KaRUZ{l8U zGI7FzM<3obs>?qjWH0a`X{lz(K7pp8##2C%xabk8yEX~CrlTY{^eVH1wh)z-On zwxlhYMx@Lu+kB1;Xi(2L;HB@hlu421cJ8d^>=a*kt{rt*a!@wS-@rG>UYhPNE9cG6&%{<_P@0u`mQU=L` zf5Uc6S3mp0nip4(88>dil!;Te*ld%D6Hr(1P!4$_q*CZDDrIYFkTW+S zI9{jMS_8u3ED9SO39E0U-0--76aGgC#5GGlWQQ*eguSrpx#ypHcJ*`5uYYy@gvk?M zKXu~dEjKUVygISU$`=TXRy)&cE#R?w6>v1^$wrHXz2t<)+hW_TZFf?u0Mp3gUa=BP z_#?8Xk)ZO6G;hOhA%4cLg04o-vlaF$ve#PhAIJtTxMDx;rio`UE5t6AG-|`!C(iC9 zGdS#?H?N;M`SnvL`MSN3(Igz~wVZV95`9|l*DZZ^@TMw;7;F=v2XTTNX#X|G~=ORh1rITfXjgvd$e$?To2 zLgp=sx{tC<1(|^B8H|_5rv!n2I*zkaO2#XBfW_d>mBo}0%Tb-*QPmP&O?Nh{XT~ep zZ(=SfU$4(DsgyKP2mrq_)n~9n=~~gThMbaVo3!0x!GlSVd@TbO>RZ@i%?VR>D8vE^ zg69!3GS!VMCA*w7Q7pY4#0E0CQBhXeBAFPnFy9oY;V zmylN>m{%q{N@G@`lmZ0+_hbMGO0!tamE{aGl+9*s`JL*Kc_ury+Za^Rz}tV>7Zgo$=GYC1!VPZ;2|rcYO7#yYA~+l2^sZ$NZeFP;2 zjJk+7QhIx6*@YD9Ig2Oy+M%fiDxFm<4sN}sQd)_K(P0@&0ytkrNE>`rs^Vp%Fykq# z^PEQbE&w#M7h6x0I#UM2WUwmfj_sBn3-wXyUj`z9ZQwaL{jN(;nNtR4)viN)c+I}- z0s$8s#NfAjKRD}UbP8jkFhER1B`z2?jA51>Jd*$pm~6(bw^$zcz*nq=lpGjVn@UG5 zYjSG3^Z{%mMV^SeY`oni`DOR?z?B{7RD1Hfk@c`ahwO29ZYH)i?Om0%+QwCGGhy;g!|oy?HkV}>b= z3Ddb>kDd#Z0RztfCN^Ca;5-eIF%V8OIs=8lG+3RtSY99=*6dG^YUZ5&323geBnqd5 zaNIr-`z#g_Clf{6kSiQ}=AL}rpKTKY#9d-XGgMxa5`?66<}zg^cL8|HGYKBY+EHil zg3(fNRqPTwgDHcoSV`11s4@jv-&04E0(nh>H51uBB5)$IX|!b_l!mK_YVs_)@~1H$ zoY^yQE(B3LTtmQ|q}WYN8UiFYI-*R6G=I&EU9a`nZXFP++SGC(3vqRZQg2X(hGJFX z#z`{AY#C)O+T_|yOM7|f;;tz+?5)DI0vFxG4WE}V}%xM6pO0tk;K* zrj)duTed-d2>gqF)BONu4tWihGg*?O!ud7A_0qP*IrvueeoptVr zWRYW|QxDXKpbRmKg(A9&#p zxAykwFsTI{x!g8tyc92FHRfE(rK?W_-ZdC2mDHXzA32n~UR@c&iWNl%FRjB~P1Vw{ z+=^0J!G6>$u@HKoI=&`VEmtIURnji09N~asffSrDKpNSBHO;JflutPe38!KHQ)sxkw zz!5Yl6r>tf>3$!y`d`3if|6rR*B792)l!auu&gMAJmEN$W$0n7K7_m~aj*a_K*4Xx z2?<%2h^2iRlwM_{PwdlJ@;ni29w%G#Q#PM53!1aoSD7PN)G@}Yy5 zpJ_sdkHtcp!RC-~hKsFsZZimiF_cgvtR@P01^LF}L>DnQ-4ZKR3Ub1$Qm)G8q?Tfq zh%AzjorFQa3l3W3V{mO4*eR1?rC6CjK1kCm!3b6e`IVDqK-7fbor9x0$yZVYhFOZ1 zke2C@P>BV0EEts^uuQMUC+8GJ790*MeNq|Dilu@aPwm_qE910@5}x_AMoS?zwkmC! zJbM}n?UoV{6ri{#WPKTp3ggTqhTyj!7T~Et*(emt4F}9j#mRjzkt2A>5);ytW-T+O zc+x+tl{g_dV`4-FQlXV;la6hyb&S#EsnvuCaHG?zjYY9gqCv^#z(CzNRm?&ycxS1x z@sWS}lKey+wPUWBG++xe+#5k!V4sqzZPmDO$VR7FQ9SrEFBSb=nI=CFIoITm0B}vy zkRHtgEnw45VbX`ysRp-0$SVPs0BQdwHb&V<6${0nm((s(fZ9Q~B0Bl(A)(_#(mPuq z&zm>aNwxtpi5sZ`^(L&ct~^g6lU7o2u;s>gzSANtp~Q=Dm|rs~Lv6dfDz>%07*qoM6N<$f+*|{UH||9 literal 0 HcmV?d00001 diff --git a/src/app.ts b/src/app.ts index a449e0f..51bb134 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,8 +1,10 @@ import { Hono } from 'hono'; import type { DB } from './db/db.js'; import type { Config } from './config.js'; -import { authMiddleware, type AppEnv } from './auth/middleware.js'; +import { sessionOrKeyAuth, type AppEnv } from './auth/middleware.js'; import { kosyncRoutes } from './routes/kosync.js'; +import { authRoutes } from './routes/auth.js'; +import { webRoutes } from './routes/web.js'; import { progressRoutes } from './routes/v1/progress.js'; import { bookmarkRoutes } from './routes/v1/bookmarks.js'; import { clippingRoutes } from './routes/v1/clippings.js'; @@ -24,13 +26,20 @@ export function createApp(db: DB, config: Config, opts: AppOptions = {}): Hono c.json({ status: 'ok', version: VERSION })); + // Web UI (landing / account pages). + app.route('/', webRoutes()); + // kosync-compatible API at the root — stock KOReader and current CrossPoint // firmware work by changing only the server URL. app.route('/', kosyncRoutes(db, config)); - // Extended CrossPoint API; same auth headers. + // Web account session auth (browser signup/login). + app.route('/auth', authRoutes(db, config)); + + // Extended CrossPoint API; accepts either the web session cookie or the + // device x-auth headers. const v1 = new Hono(); - v1.use('*', authMiddleware(db)); + v1.use('*', sessionOrKeyAuth(db)); v1.route('/', progressRoutes(db)); v1.route('/', bookmarkRoutes(db)); v1.route('/', clippingRoutes(db)); diff --git a/src/auth/middleware.ts b/src/auth/middleware.ts index 11e7798..9c6f8d9 100644 --- a/src/auth/middleware.ts +++ b/src/auth/middleware.ts @@ -1,6 +1,8 @@ import type { Context, MiddlewareHandler } from 'hono'; +import { getCookie } from 'hono/cookie'; import type { DB } from '../db/db.js'; import { verifyKey } from './password.js'; +import { SESSION_COOKIE, verifySession } from './session.js'; export interface AuthedUser { id: number; @@ -54,6 +56,28 @@ export function authMiddleware(db: DB): MiddlewareHandler { }; } +/** + * Accept EITHER a web session cookie OR the device x-auth headers. Used for the + * /api/v1 surface so both the browser (cookie) and the firmware (headers) reach + * the same endpoints. The cookie path is checked first and is cheap (HMAC, no + * PBKDF2); falls through to header auth when absent. + */ +export function sessionOrKeyAuth(db: DB): MiddlewareHandler { + const headerAuth = authMiddleware(db); + const getById = db.prepare('SELECT id, username FROM users WHERE id = ?'); + return async (c, next) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (session) { + const row = getById.get(session.uid) as { id: number; username: string } | undefined; + if (row) { + c.set('user', { id: row.id, username: row.username }); + return next(); + } + } + return headerAuth(c, next); + }; +} + /** Minimal in-memory per-IP fixed-window rate limiter. */ export function rateLimiter(limitPerMinute: number): MiddlewareHandler { const hits = new Map(); diff --git a/src/auth/session.ts b/src/auth/session.ts new file mode 100644 index 0000000..813f072 --- /dev/null +++ b/src/auth/session.ts @@ -0,0 +1,85 @@ +import crypto from 'node:crypto'; + +/** + * Stateless signed session cookies for the web account. Format: + * base64url(JSON{uid, exp}) + '.' + base64url(HMAC-SHA256(payload)) + * No session table — the HMAC makes the cookie unforgeable. Signed with + * SESSION_SECRET (falls back to TOKEN_ENC_KEY, then an ephemeral per-process key + * so zero-config still works, at the cost of sessions dropping on restart). + */ + +const COOKIE_NAME = 'cp_session'; +const DEFAULT_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days + +let cachedSecret: Buffer | undefined; +let warnedEphemeral = false; + +function sessionSecret(env: NodeJS.ProcessEnv = process.env): Buffer { + if (cachedSecret) return cachedSecret; + const raw = env.SESSION_SECRET || env.TOKEN_ENC_KEY; + if (raw) { + cachedSecret = crypto.createHash('sha256').update(raw).digest(); + } else { + cachedSecret = crypto.randomBytes(32); + if (!warnedEphemeral) { + warnedEphemeral = true; + console.warn( + JSON.stringify({ + msg: 'no SESSION_SECRET/TOKEN_ENC_KEY set; using an ephemeral key — sessions drop on restart', + }) + ); + } + } + return cachedSecret; +} + +export function resetSessionSecretCache(): void { + cachedSecret = undefined; +} + +export const SESSION_COOKIE = COOKIE_NAME; + +function b64url(buf: Buffer): string { + return buf.toString('base64url'); +} + +export function signSession( + userId: number, + ttlSeconds = DEFAULT_TTL_SECONDS, + env: NodeJS.ProcessEnv = process.env +): string { + const exp = Math.floor(Date.now() / 1000) + ttlSeconds; + const payload = b64url(Buffer.from(JSON.stringify({ uid: userId, exp }))); + const sig = b64url(crypto.createHmac('sha256', sessionSecret(env)).update(payload).digest()); + return `${payload}.${sig}`; +} + +export function verifySession( + token: string | undefined, + env: NodeJS.ProcessEnv = process.env +): { uid: number } | null { + if (!token) return null; + const dot = token.indexOf('.'); + if (dot <= 0) return null; + const payload = token.slice(0, dot); + const sig = token.slice(dot + 1); + const expected = b64url( + crypto.createHmac('sha256', sessionSecret(env)).update(payload).digest() + ); + const a = Buffer.from(sig); + const b = Buffer.from(expected); + if (a.length !== b.length || !crypto.timingSafeEqual(a, b)) return null; + try { + const data = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { + uid: number; + exp: number; + }; + if (typeof data.uid !== 'number' || typeof data.exp !== 'number') return null; + if (data.exp < Math.floor(Date.now() / 1000)) return null; + return { uid: data.uid }; + } catch { + return null; + } +} + +export const SESSION_TTL_SECONDS = DEFAULT_TTL_SECONDS; diff --git a/src/routes/auth.ts b/src/routes/auth.ts new file mode 100644 index 0000000..87abbdf --- /dev/null +++ b/src/routes/auth.ts @@ -0,0 +1,137 @@ +import crypto from 'node:crypto'; +import { Hono, type Context } from 'hono'; +import { getCookie, setCookie, deleteCookie } from 'hono/cookie'; +import type { DB } from '../db/db.js'; +import type { Config } from '../config.js'; +import type { AppEnv } from '../auth/middleware.js'; +import { hashKey, verifyKey } from '../auth/password.js'; +import { rateLimiter } from '../auth/middleware.js'; +import { + SESSION_COOKIE, + SESSION_TTL_SECONDS, + signSession, + verifySession, +} from '../auth/session.js'; +import { nowSeconds } from '../models/sync.js'; +import { USERNAME_RE } from './kosync.js'; + +/** + * Token-based web accounts. There are no passwords: signup issues a generated + * token (shown once) that IS the credential. The device sends it as the kosync + * secret (username + token, MD5'd by the client); the web pastes the token to + * establish a session cookie. We store PBKDF2(MD5(token)) in users.key_hash — + * the exact shape device auth already verifies — so one secret works for both. + * + * Token format: xp1__. The embedded id lets the web log in from + * the token alone (no username needed); security is the random secret + hash. + */ + +const TOKEN_RE = /^xp1_(\d+)_[0-9a-f]{32}$/; + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +function makeToken(userId: number): string { + return `xp1_${userId}_${crypto.randomBytes(16).toString('hex')}`; +} + +function setSessionCookie(c: Context, userId: number) { + const proto = c.req.header('x-forwarded-proto'); + const secure = proto ? proto.split(',')[0].trim() === 'https' : false; + setCookie(c, SESSION_COOKIE, signSession(userId), { + httpOnly: true, + sameSite: 'Lax', + secure, + path: '/', + maxAge: SESSION_TTL_SECONDS, + }); +} + +export function authRoutes(db: DB, config: Config): Hono { + const app = new Hono(); + + // Create an account: pick a username, receive a token (shown once). + app.post('/signup', rateLimiter(config.authRateLimitPerMinute), async (c) => { + if (config.registrationDisabled) { + return c.json({ error: 'Registration is disabled' }, 403); + } + let username: string | null = null; + try { + const body = (await c.req.json()) as Record; + username = typeof body.username === 'string' ? body.username.trim() : null; + } catch { + /* fall through to validation error */ + } + if (!username || !USERNAME_RE.test(username)) { + return c.json({ error: 'Invalid username' }, 400); + } + if (db.prepare('SELECT 1 FROM users WHERE username = ?').get(username)) { + return c.json({ error: 'Username is already registered' }, 409); + } + // Insert to get the id, then bake it into the token and store its hash. + const token = crypto.randomBytes(16).toString('hex'); + const info = db + .prepare('INSERT INTO users (username, key_hash, created_at) VALUES (?, ?, ?)') + .run(username, '', nowSeconds()); + const userId = Number(info.lastInsertRowid); + const fullToken = `xp1_${userId}_${token}`; + db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run(hashKey(md5(fullToken)), userId); + setSessionCookie(c, userId); + return c.json({ username, token: fullToken }); + }); + + // Web login: paste the token, get a session cookie. + app.post('/login', rateLimiter(config.authRateLimitPerMinute), async (c) => { + let token: string | null = null; + try { + const body = (await c.req.json()) as Record; + token = typeof body.token === 'string' ? body.token.trim() : null; + } catch { + /* fall through */ + } + const parsed = token?.match(TOKEN_RE); + if (!token || !parsed) return c.json({ error: 'Invalid token' }, 401); + const userId = Number(parsed[1]); + const row = db + .prepare('SELECT key_hash FROM users WHERE id = ?') + .get(userId) as { key_hash: string } | undefined; + if (!row || !verifyKey(md5(token), row.key_hash)) { + return c.json({ error: 'Invalid token' }, 401); + } + setSessionCookie(c, userId); + return c.json({ ok: true }); + }); + + app.post('/logout', (c) => { + deleteCookie(c, SESSION_COOKIE, { path: '/' }); + return c.json({ ok: true }); + }); + + app.get('/me', (c) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return c.json({ error: 'Not signed in' }, 401); + const row = db + .prepare('SELECT username FROM users WHERE id = ?') + .get(session.uid) as { username: string } | undefined; + if (!row) return c.json({ error: 'Not signed in' }, 401); + return c.json({ username: row.username }); + }); + + // Rotate the token (revokes the old one — the device must be updated). + // Requires an active web session. + app.post('/token/rotate', (c) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return c.json({ error: 'Not signed in' }, 401); + const exists = db.prepare('SELECT 1 FROM users WHERE id = ?').get(session.uid); + if (!exists) return c.json({ error: 'Not signed in' }, 401); + const fullToken = makeToken(session.uid); + db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run( + hashKey(md5(fullToken)), + session.uid + ); + return c.json({ token: fullToken }); + }); + + return app; +} diff --git a/src/routes/kosync.ts b/src/routes/kosync.ts index 30f7772..869fe69 100644 --- a/src/routes/kosync.ts +++ b/src/routes/kosync.ts @@ -13,7 +13,7 @@ import { parsePosition } from '../models/position.js'; import { nowSeconds } from '../models/sync.js'; import { fanOutProgress } from '../connectors/fanout.js'; -const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; +export const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; export function isValidDocument(v: unknown): v is string { // KOReader sends a 32-hex MD5, but the key is opaque — stay lenient. diff --git a/src/routes/web.ts b/src/routes/web.ts new file mode 100644 index 0000000..501f69e --- /dev/null +++ b/src/routes/web.ts @@ -0,0 +1,307 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Hono } from 'hono'; +import { getCookie } from 'hono/cookie'; +import type { AppEnv } from '../auth/middleware.js'; +import { SESSION_COOKIE, verifySession } from '../auth/session.js'; + +/** + * Minimal server-rendered web UI (no framework, no build step, no deps). Styled + * to match the CrossPoint Reader site: Inter (UI) + Lora (display) + Geist Mono, + * stone neutrals with the forest-green brand accent, white cards on stone-50. + * Pages are static HTML calling the JSON /auth and /api/v1 endpoints via fetch; + * the session cookie authenticates automatically (same-origin). + */ + +const ASSETS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'assets'); +const LOGO = fs.readFileSync(path.join(ASSETS_DIR, 'logo.png')); +const FAVICON = fs.readFileSync(path.join(ASSETS_DIR, 'favicon.png')); + +const STYLE = ` + :root { + color-scheme: light; + --stone-50:#fafaf9; --stone-100:#f5f5f4; --stone-200:#e7e5e4; --stone-300:#d6d3d1; + --stone-400:#a8a29e; --stone-500:#78716c; --stone-600:#57534e; --stone-700:#44403c; + --stone-900:#1c1917; --stone-950:#0c0a09; + --brand-50:#f0f5f3; --brand-100:#d6e5de; --brand-400:#69917d; --brand-500:#4a7a62; + --brand-600:#3d6652; --brand-700:#315243; + --ring:rgba(12,10,9,0.05); + } + * { box-sizing:border-box; } + html { -webkit-text-size-adjust:100%; } + body { + margin:0; background:var(--stone-50); color:var(--stone-900); + font-family:"InterVariable","Inter",ui-sans-serif,system-ui,sans-serif; + font-feature-settings:"cv02","cv03","cv04","cv11"; + -webkit-font-smoothing:antialiased; + } + .display { font-family:"Lora",ui-serif,serif; } + .mono { font-family:"Geist Mono",ui-monospace,monospace; } + + header.site { + position:sticky; top:0; z-index:40; + border-bottom:1px solid rgba(231,229,228,0.8); + background:rgba(250,250,249,0.8); backdrop-filter:blur(12px); + } + header.site .bar { max-width:64rem; margin:0 auto; padding:14px 20px; + display:flex; align-items:center; justify-content:space-between; gap:16px; } + .wordmark { display:flex; align-items:center; gap:10px; text-decoration:none; } + .wordmark img { width:28px; height:28px; border-radius:6px; display:block; } + .wordmark span { font-family:"Lora",serif; font-weight:600; font-size:16px; + letter-spacing:-0.01em; color:var(--stone-900); } + .wordmark .accent { color:var(--brand-600); } + + .wrap { max-width:32rem; margin:0 auto; padding:44px 20px 96px; } + .center { text-align:center; } + .eyebrow { display:inline-flex; align-items:center; gap:7px; font-size:12px; font-weight:600; + letter-spacing:0.08em; text-transform:uppercase; color:var(--brand-600); } + .eyebrow::before { content:""; width:16px; height:1px; background:var(--brand-400); display:inline-block; } + h1 { font-family:"Lora",serif; font-weight:600; font-size:30px; line-height:1.15; + letter-spacing:-0.015em; margin:14px 0 0; color:var(--stone-900); } + .sub { color:var(--stone-600); margin:12px 0 0; line-height:1.6; } + + .card { background:#fff; border-radius:12px; box-shadow:0 0 0 1px var(--ring); padding:24px; } + .card + .card { margin-top:16px; } + .card h2 { font-size:13px; font-weight:600; text-transform:uppercase; letter-spacing:0.05em; + color:var(--stone-500); margin:0 0 14px; } + + label { display:block; font-size:14px; font-weight:500; color:var(--stone-700); margin:0 0 6px; } + input { width:100%; padding:10px 14px; border:1px solid var(--stone-200); border-radius:8px; + background:var(--stone-50); color:var(--stone-900); font-size:14px; } + input::placeholder { color:var(--stone-400); } + input:focus { outline:none; border-color:var(--brand-400); box-shadow:0 0 0 3px rgba(74,122,98,0.15); } + + button { appearance:none; border:0; border-radius:8px; padding:10px 16px; font-size:14px; + font-weight:600; cursor:pointer; font-family:inherit; } + button.primary { background:var(--brand-500); color:#fff; box-shadow:0 1px 2px rgba(0,0,0,0.06); } + button.primary:hover { background:var(--brand-600); } + button.ghost { background:#fff; color:var(--stone-700); box-shadow:0 0 0 1px var(--stone-200); } + button.ghost:hover { background:var(--stone-50); } + button.copied { color:var(--brand-700); box-shadow:0 0 0 1px var(--brand-100); background:var(--brand-50); opacity:1; } + button:disabled { opacity:.5; cursor:default; } + button.copied:disabled { opacity:1; } + button.full { width:100%; } + .mt { margin-top:14px; } + + .row { display:flex; align-items:center; justify-content:space-between; gap:14px; } + .muted { color:var(--stone-500); font-size:13px; line-height:1.55; } + .pill { font-size:11px; font-weight:600; padding:2px 9px; border-radius:999px; + box-shadow:0 0 0 1px var(--stone-200); color:var(--stone-500); text-transform:uppercase; letter-spacing:0.03em; } + .pill.ok { color:var(--brand-700); box-shadow:0 0 0 1px var(--brand-100); background:var(--brand-50); } + .pill.warn { color:#9a7a2e; box-shadow:0 0 0 1px #ecdcae; background:#faf5e6; } + + code.token { display:block; margin:12px 0; padding:12px 14px; background:var(--stone-50); + box-shadow:0 0 0 1px var(--stone-200); border-radius:8px; font-family:"Geist Mono",monospace; + font-size:13px; word-break:break-all; color:var(--stone-900); } + .notice { margin-top:16px; padding:14px 16px; background:var(--brand-50); border-radius:8px; } + .notice b { color:var(--brand-700); } + .err { color:#b91c1c; font-size:13px; min-height:18px; margin-top:8px; } + .foot { text-align:center; color:var(--stone-400); font-size:12px; margin-top:28px; } + .foot a { color:var(--stone-500); } + a { color:var(--brand-600); text-decoration:none; } + a:hover { text-decoration:underline; } +`; + +function shell(title: string, body: string): string { + return ` + +${title} · CrossPoint Sync + + + + + + +
+ CrossPoint Sync + ${title === 'Account' ? '' : ''} +
+
${body}
+`; +} + +const LANDING = shell( + 'Sign in', + `
+ Sync Hub +

Your reading, everywhere

+

One account to sync reading progress across your devices and link services like Hardcover, Readwise, and BookFusion.

+
+ +
+

Create account

+ + + +
+ +
+ +
+

Sign in

+ + + +
+
+ +

Want to self host this?

+ +` +); + +const ACCOUNT = shell( + 'Account', + `
+ Account +

Signed in as …

+
+ +

Linked services

+

Loading…

+ +

Your sync token

+
+

Use this as the password on your reader (KOReader / CrossPoint sync settings), with your username. Rotating it signs your reader out until you enter the new one.

+ + +
+ +

Want to self host this?

+ +` +); + +export function webRoutes(): Hono { + const app = new Hono(); + + app.get('/logo.png', (c) => { + c.header('content-type', 'image/png'); + c.header('cache-control', 'public, max-age=86400'); + return c.body(LOGO); + }); + app.get('/favicon.png', (c) => { + c.header('content-type', 'image/png'); + c.header('cache-control', 'public, max-age=86400'); + return c.body(FAVICON); + }); + + app.get('/', (c) => { + if (verifySession(getCookie(c, SESSION_COOKIE))) return c.redirect('/account'); + return c.html(LANDING); + }); + + app.get('/account', (c) => { + if (!verifySession(getCookie(c, SESSION_COOKIE))) return c.redirect('/'); + return c.html(ACCOUNT); + }); + + return app; +} diff --git a/test/account.test.ts b/test/account.test.ts new file mode 100644 index 0000000..42b1d68 --- /dev/null +++ b/test/account.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { makeTestApp, md5, DOC } from './helpers.js'; +import { resetSessionSecretCache } from '../src/auth/session.js'; + +type App = ReturnType['app']; + +afterEach(() => resetSessionSecretCache()); + +async function signup(app: App, username: string) { + return app.request('/auth/signup', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username }), + }); +} + +describe('token-based web account', () => { + it('signup issues a xp1_ token and logs the browser in (cookie)', async () => { + const { app } = makeTestApp(); + const res = await signup(app, 'julia'); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.username).toBe('julia'); + expect(body.token).toMatch(/^xp1_\d+_[0-9a-f]{32}$/); + // Signup set a session cookie usable immediately. + const cookie = res.headers.get('set-cookie'); + expect(cookie).toContain('cp_session='); + const me = await app.request('/auth/me', { headers: { cookie: cookie!.split(';')[0] } }); + expect((await me.json()).username).toBe('julia'); + }); + + it('the token works as the kosync device secret (username + MD5(token))', async () => { + const { app } = makeTestApp(); + const token = (await (await signup(app, 'julia')).json()).token as string; + // Device auth: x-auth-user + x-auth-key = MD5(token) + const headers = { 'x-auth-user': 'julia', 'x-auth-key': md5(token) }; + const auth = await app.request('/users/auth', { headers }); + expect(auth.status).toBe(200); + expect(await auth.json()).toEqual({ authorized: 'OK' }); + // And it can drive a real v1 call. + const put = await app.request('/api/v1/documents', { + method: 'PUT', + headers: { ...headers, 'content-type': 'application/json' }, + body: JSON.stringify({ items: [{ document: DOC, title: 'X' }] }), + }); + expect(put.status).toBe(200); + }); + + it('web login with the token alone establishes a session', async () => { + const { app } = makeTestApp(); + const token = (await (await signup(app, 'julia')).json()).token as string; + const login = await app.request('/auth/login', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ token }), + }); + expect(login.status).toBe(200); + const cookie = login.headers.get('set-cookie')!.split(';')[0]; + // Session cookie reaches v1 endpoints without x-auth headers. + const list = await app.request('/api/v1/connectors', { headers: { cookie } }); + expect(list.status).toBe(200); + }); + + it('rejects a bad or malformed token', async () => { + const { app } = makeTestApp(); + await signup(app, 'julia'); + for (const token of ['garbage', 'xp1_1_deadbeef', 'xp1_999_' + 'a'.repeat(32)]) { + const res = await app.request('/auth/login', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ token }), + }); + expect(res.status).toBe(401); + } + }); + + it('rotate issues a new token and revokes the old one', async () => { + const { app } = makeTestApp(); + const first = (await (await signup(app, 'julia')).json()).token as string; + const cookie = (await signupCookie(app, 'julia2')); + // rotate for julia2 + const rot = await app.request('/auth/token/rotate', { method: 'POST', headers: { cookie } }); + expect(rot.status).toBe(200); + const newToken = (await rot.json()).token as string; + expect(newToken).toMatch(/^xp1_\d+_[0-9a-f]{32}$/); + + // New token authenticates as the device; old first-user token is unaffected. + const okNew = await app.request('/users/auth', { + headers: { 'x-auth-user': 'julia2', 'x-auth-key': md5(newToken) }, + }); + expect(okNew.status).toBe(200); + expect(first).not.toBe(newToken); + }); + + it('respects REGISTRATION_DISABLED', async () => { + const { app } = makeTestApp({ registrationDisabled: true }); + expect((await signup(app, 'nope')).status).toBe(403); + }); + + it('rejects duplicate usernames and invalid usernames', async () => { + const { app } = makeTestApp(); + await signup(app, 'julia'); + expect((await signup(app, 'julia')).status).toBe(409); + expect((await signup(app, 'has spaces')).status).toBe(400); + }); +}); + +async function signupCookie(app: App, username: string): Promise { + const res = await app.request('/auth/signup', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username }), + }); + return res.headers.get('set-cookie')!.split(';')[0]; +} From 96c5aa2050be748d8ebf590d7b9baa6322fa3181 Mon Sep 17 00:00:00 2001 From: Justin Mitchell Date: Fri, 7 Aug 2026 03:05:48 -0400 Subject: [PATCH 3/4] Add master account system for web login Introduces a separate accounts table for website authentication, distinct from kosync device credentials. Kosync users can be linked to a master account via account_id foreign key, allowing orphaned kosync accounts to be claimed later. One kosync user per master account enforced by unique index. --- migrations/0004_master_accounts.sql | 16 ++ src/app.ts | 6 +- src/auth/middleware.ts | 53 +++++- src/auth/session.ts | 4 +- src/connectors/bookfusion.ts | 201 +++++++++++++++++++++++ src/connectors/fanout.ts | 14 +- src/connectors/kosync.ts | 129 +++++++++++++++ src/connectors/matching.ts | 8 +- src/connectors/readwise.ts | 8 +- src/connectors/registry.ts | 11 +- src/connectors/runner.ts | 2 +- src/connectors/types.ts | 26 ++- src/crypto/secrets.ts | 2 +- src/index.ts | 4 +- src/models/position.ts | 2 +- src/models/stats.ts | 2 +- src/routes/account.ts | 180 +++++++++++++++++++++ src/routes/auth.ts | 91 ++++++----- src/routes/kosync.ts | 4 +- src/routes/v1/connectors.ts | 56 ++++++- src/routes/v1/progress.ts | 4 +- src/routes/web.ts | 243 ++++++++++++++++++++++++---- test/account.test.ts | 240 +++++++++++++++++++-------- test/connectors-more.test.ts | 146 +++++++++++++++++ test/connectors.test.ts | 2 +- 25 files changed, 1273 insertions(+), 181 deletions(-) create mode 100644 migrations/0004_master_accounts.sql create mode 100644 src/connectors/bookfusion.ts create mode 100644 src/connectors/kosync.ts create mode 100644 src/routes/account.ts create mode 100644 test/connectors-more.test.ts diff --git a/migrations/0004_master_accounts.sql b/migrations/0004_master_accounts.sql new file mode 100644 index 0000000..aba2c3a --- /dev/null +++ b/migrations/0004_master_accounts.sql @@ -0,0 +1,16 @@ +-- Master ("general login") accounts. The website identity you sign into, kept +-- separate from the kosync sync credential your reader uses. A kosync account +-- (users row) is linked to a master account as its native sync identity; the +-- reading data and connectors stay keyed to the kosync user. See docs/design. + +CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + handle TEXT NOT NULL UNIQUE, + token_hash TEXT NOT NULL, -- PBKDF2(MD5(master token)) + created_at INTEGER NOT NULL +); + +-- Link a kosync user to its owning master account. NULL = orphan kosync account +-- (e.g. created directly on a device), linkable later. One kosync per master. +ALTER TABLE users ADD COLUMN account_id INTEGER REFERENCES accounts(id); +CREATE UNIQUE INDEX idx_users_account ON users(account_id) WHERE account_id IS NOT NULL; diff --git a/src/app.ts b/src/app.ts index 51bb134..b33e71e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,6 +4,7 @@ import type { Config } from './config.js'; import { sessionOrKeyAuth, type AppEnv } from './auth/middleware.js'; import { kosyncRoutes } from './routes/kosync.js'; import { authRoutes } from './routes/auth.js'; +import { accountRoutes } from './routes/account.js'; import { webRoutes } from './routes/web.js'; import { progressRoutes } from './routes/v1/progress.js'; import { bookmarkRoutes } from './routes/v1/bookmarks.js'; @@ -29,12 +30,13 @@ export function createApp(db: DB, config: Config, opts: AppOptions = {}): Hono { } /** - * Accept EITHER a web session cookie OR the device x-auth headers. Used for the - * /api/v1 surface so both the browser (cookie) and the firmware (headers) reach - * the same endpoints. The cookie path is checked first and is cheap (HMAC, no - * PBKDF2); falls through to header auth when absent. + * Require a valid master ("general login") session cookie. Sets `account`. + * Used for the /account management surface (kosync link, master token rotate). + */ +export function masterAuth(db: DB): MiddlewareHandler { + const getAccount = db.prepare('SELECT id, handle FROM accounts WHERE id = ?'); + return async (c, next) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return kosyncError(c, 401, 2001, 'Not signed in'); + const row = getAccount.get(session.uid) as { id: number; handle: string } | undefined; + if (!row) return kosyncError(c, 401, 2001, 'Not signed in'); + c.set('account', { id: row.id, handle: row.handle }); + await next(); + }; +} + +/** + * Accept EITHER a web session cookie OR the device x-auth headers, resolving to + * the kosync sync identity that owns the reading data. Used for the /api/v1 + * surface so both the browser and the firmware reach the same endpoints: + * - device: x-auth headers -> the kosync user directly. + * - web: master session cookie -> the account's linked native kosync user. + * A signed-in master account with no kosync account linked yet gets 409 (the + * web UI prompts to create/link one before showing data). */ export function sessionOrKeyAuth(db: DB): MiddlewareHandler { const headerAuth = authMiddleware(db); - const getById = db.prepare('SELECT id, username FROM users WHERE id = ?'); + const getAccount = db.prepare('SELECT id, handle FROM accounts WHERE id = ?'); + const getKosyncForAccount = db.prepare( + 'SELECT id, username FROM users WHERE account_id = ?' + ); return async (c, next) => { const session = verifySession(getCookie(c, SESSION_COOKIE)); if (session) { - const row = getById.get(session.uid) as { id: number; username: string } | undefined; - if (row) { - c.set('user', { id: row.id, username: row.username }); + const account = getAccount.get(session.uid) as { id: number; handle: string } | undefined; + if (account) { + c.set('account', account); + const kosync = getKosyncForAccount.get(account.id) as + | { id: number; username: string } + | undefined; + if (!kosync) { + return c.json({ code: 2005, message: 'No sync account linked' }, 409); + } + c.set('user', kosync); return next(); } } diff --git a/src/auth/session.ts b/src/auth/session.ts index 813f072..4ad4635 100644 --- a/src/auth/session.ts +++ b/src/auth/session.ts @@ -3,7 +3,7 @@ import crypto from 'node:crypto'; /** * Stateless signed session cookies for the web account. Format: * base64url(JSON{uid, exp}) + '.' + base64url(HMAC-SHA256(payload)) - * No session table — the HMAC makes the cookie unforgeable. Signed with + * No session table - the HMAC makes the cookie unforgeable. Signed with * SESSION_SECRET (falls back to TOKEN_ENC_KEY, then an ephemeral per-process key * so zero-config still works, at the cost of sessions dropping on restart). */ @@ -25,7 +25,7 @@ function sessionSecret(env: NodeJS.ProcessEnv = process.env): Buffer { warnedEphemeral = true; console.warn( JSON.stringify({ - msg: 'no SESSION_SECRET/TOKEN_ENC_KEY set; using an ephemeral key — sessions drop on restart', + msg: 'no SESSION_SECRET/TOKEN_ENC_KEY set; using an ephemeral key; sessions drop on restart', }) ); } diff --git a/src/connectors/bookfusion.ts b/src/connectors/bookfusion.ts new file mode 100644 index 0000000..724ac30 --- /dev/null +++ b/src/connectors/bookfusion.ts @@ -0,0 +1,201 @@ +import { decideMatch, extractTitleAuthor, type Candidate } from './matching.js'; +import type { + Connector, + Credential, + DeviceLinkPoll, + DeviceLinkStart, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * BookFusion connector (Tier 3, experimental). Uses BookFusion's OAuth 2.0 + * device-authorization grant to obtain a per-user access token, then pushes + * reading position to /api/user/books/{id}/reading_position. Endpoints and the + * api_version header come from BookFusion's official KOReader plugin. + * + * !!! LIVE-VERIFY GATE !!! + * BookFusion's API is real but not formally public (a developer portal is "coming"). + * The plugin hardcodes client_id "koreader"; confirm we may reuse it or register + * our own before shipping. Reconfirm the device-code, search, and reading_position + * shapes against a live account. Search this file for GATE. + */ + +const BASE = 'https://www.bookfusion.com'; +const API_VERSION = 'application/json; api_version=10'; +// GATE: confirm we can use this client_id; BookFusion may require registration. +const CLIENT_ID = 'koreader'; + +interface BookFusionCred extends Credential { + access_token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as BookFusionCred).access_token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing bookfusion token'); + return t; +} + +function authHeaders(token: string): Record { + return { authorization: `Bearer ${token}`, accept: API_VERSION, 'content-type': 'application/json' }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + // GATE: confirm a cheap authenticated endpoint for validation. + const res = await http(`${BASE}/api/user/books/search`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ query: '', per_page: 1 }), + }); + if (res.status === 401 || res.status === 403) return { ok: false, error: 'invalid token' }; + if (res.status >= 200 && res.status < 300) return { ok: true }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +async function match( + cred: Credential, + doc: DocumentMeta, + http: HttpTransport +): Promise { + const ta = extractTitleAuthor(doc); + if (!ta) return null; + const token = tokenOf(cred); + const q = `${ta.title} ${ta.author}`.trim(); + // GATE: confirm the search request/response shape. + const res = await http(`${BASE}/api/user/books/search`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ query: q, per_page: 10 }), + }); + if (res.status < 200 || res.status >= 300) return null; + let body: any; + try { + body = await res.json(); + } catch { + return null; + } + const hits = extractBooks(body); + if (hits.length === 0) return null; + const decision = decideMatch(ta.title, ta.author, hits); + if (!decision.accepted || !decision.best) return null; + return { externalId: decision.best.externalId, confidence: decision.best.score, queryUsed: q }; +} + +/** GATE: adapt to the real search payload. */ +export function extractBooks(body: any): Candidate[] { + const arr = Array.isArray(body) ? body : (body?.books ?? body?.results ?? []); + const out: Candidate[] = []; + for (const b of Array.isArray(arr) ? arr : []) { + const id = b?.id ?? b?.book_id; + const title = b?.title; + if (id == null || typeof title !== 'string') continue; + const author = b?.author ?? (Array.isArray(b?.authors) ? b.authors[0]?.name ?? b.authors[0] : undefined); + out.push({ externalId: String(id), title, author }); + } + return out; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const token = tokenOf(cred); + const percentage = Math.max(0, Math.min(1, ev.percentage ?? 0)) * 100; // BookFusion uses 0..100 + // GATE: confirm reading_position field names (percentage, page_position_in_book, cfi). + const res = await http(`${BASE}/api/user/books/${m.externalId}/reading_position`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ percentage: Number(percentage.toFixed(4)) }), + }); + if (res.status === 401 || res.status === 403) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + if (res.status >= 200 && res.status < 300) return { ok: true }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; +} + +// --- Device-code linking flow ------------------------------------------------- + +async function beginLink(http: HttpTransport): Promise { + const res = await http(`${BASE}/api/user/auth/device`, { + method: 'POST', + headers: { accept: API_VERSION, 'content-type': 'application/json' }, + body: JSON.stringify({ client_id: CLIENT_ID }), + }); + const body = (await res.json()) as { + device_code: string; + user_code: string; + verification_uri: string; + interval?: number; + expires_in?: number; + }; + return { + deviceCode: body.device_code, + userCode: body.user_code, + verificationUri: body.verification_uri, + interval: body.interval ?? 5, + expiresIn: body.expires_in ?? 900, + }; +} + +async function pollLink(deviceCode: string, http: HttpTransport): Promise { + const res = await http(`${BASE}/api/user/auth/token`, { + method: 'POST', + headers: { accept: API_VERSION, 'content-type': 'application/json' }, + body: JSON.stringify({ + grant_type: 'urn:ietf:params:oauth:grant-type:device_code', + client_id: CLIENT_ID, + device_code: deviceCode, + }), + }); + let body: any = {}; + try { + body = await res.json(); + } catch { + /* ignore */ + } + if (body?.access_token) { + return { status: 'ok', credential: { access_token: body.access_token } }; + } + switch (body?.error) { + case 'authorization_pending': + case 'slow_down': + return { status: 'pending' }; + case 'access_denied': + return { status: 'denied', error: 'You declined the request on BookFusion.' }; + case 'expired_token': + return { status: 'expired', error: 'The code expired. Start again.' }; + default: + return res.status >= 500 + ? { status: 'pending' } + : { status: 'error', error: body?.error ?? `status ${res.status}` }; + } +} + +export const bookfusionConnector: Connector = { + id: 'bookfusion', + displayName: 'BookFusion', + tier: 3, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'device_code', + experimental: true, + validate, + match, + push, + beginLink, + pollLink, +}; diff --git a/src/connectors/fanout.ts b/src/connectors/fanout.ts index e8d2dff..a690e43 100644 --- a/src/connectors/fanout.ts +++ b/src/connectors/fanout.ts @@ -30,13 +30,25 @@ export function fanOutProgress( userId: number, document: string, percentage: number, - timestamp: number + timestamp: number, + progress?: string, + positionJson?: string | null ): void { const finished = percentage >= 0.98; + let position: Record | null = null; + if (positionJson) { + try { + position = JSON.parse(positionJson) as Record; + } catch { + position = null; + } + } fanOut(db, userId, { kind: finished ? 'finished' : 'progress', document, percentage, + progress, + position, timestamp, }); } diff --git a/src/connectors/kosync.ts b/src/connectors/kosync.ts new file mode 100644 index 0000000..0524797 --- /dev/null +++ b/src/connectors/kosync.ts @@ -0,0 +1,129 @@ +import crypto from 'node:crypto'; +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * External kosync mirror connector. Forwards reading progress to ANOTHER kosync + * server (e.g. sync.koreader.rocks, a friend's server, or a second CrossPoint + * Sync instance) so other KOReader devices see it too. + * + * This is the simplest connector: kosync keys progress by the same 32-hex + * document hash we already have, so there is NO book matching. It also speaks + * the exact protocol we store, so mirroring is lossless (we can forward the rich + * position + metadata superset; a plain kosync target ignores the extras). + * + * Outbound only for now (we push to them); fan-in would need loop suppression. + */ + +interface KosyncCred extends Credential { + server: string; + username: string; + password: string; +} + +function parseCred(cred: Credential): KosyncCred | null { + const server = typeof cred.server === 'string' ? cred.server.trim() : ''; + const username = typeof cred.username === 'string' ? cred.username.trim() : ''; + const password = typeof cred.password === 'string' ? cred.password : ''; + if (!server || !username || !password) return null; + return { server, username, password }; +} + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +/** Normalize a server URL: add scheme if missing, strip trailing slashes. */ +export function baseUrl(server: string): string { + let url = server.includes('://') ? server : `https://${server}`; + while (url.endsWith('/')) url = url.slice(0, -1); + return url; +} + +function authHeaders(c: KosyncCred): Record { + return { + 'x-auth-user': c.username, + 'x-auth-key': md5(c.password), + accept: 'application/vnd.koreader.v1+json', + 'content-type': 'application/json', + }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + const c = parseCred(cred); + if (!c) return { ok: false, error: 'server, username and password are required' }; + try { + const res = await http(`${baseUrl(c.server)}/users/auth`, { + method: 'GET', + headers: authHeaders(c), + }); + if (res.status === 200) return { ok: true, accountLabel: `${c.username} @ ${c.server}` }; + if (res.status === 401) return { ok: false, error: 'invalid sync credentials' }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +// The document hash is the kosync document id: matching is identity, no network. +async function match(_cred: Credential, doc: DocumentMeta, _http: HttpTransport): Promise { + return { externalId: doc.document, confidence: 1 }; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const c = parseCred(cred); + if (!c) return { ok: false, retryable: false, needsReauth: true, error: 'bad credential' }; + + const percentage = Math.max(0, Math.min(1, ev.percentage ?? 0)); + const body: Record = { + document: m.externalId, + // kosync servers reject an empty progress string; fall back to the percentage. + progress: ev.progress && ev.progress.length > 0 ? ev.progress : String(percentage), + percentage, + device: 'CrossPoint Sync', + device_id: 'crosspoint-sync-mirror', + }; + // Forward the rich position superset (a plain kosync target ignores it). + if (ev.position) body.position = ev.position; + + try { + const res = await http(`${baseUrl(c.server)}/syncs/progress`, { + method: 'PUT', + headers: authHeaders(c), + body: JSON.stringify(body), + }); + if (res.status === 200 || res.status === 202) return { ok: true }; + if (res.status === 401) return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, retryable: true, error: err instanceof Error ? err.message : String(err) }; + } +} + +export const kosyncConnector: Connector = { + id: 'kosync', + displayName: 'Another KOSync server', + tier: 1, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'kosync', + experimental: false, + validate, + match, + push, +}; diff --git a/src/connectors/matching.ts b/src/connectors/matching.ts index 2de3b0e..718d685 100644 --- a/src/connectors/matching.ts +++ b/src/connectors/matching.ts @@ -3,7 +3,7 @@ import type { DocumentMeta } from './types.js'; /** * Shared, connector-agnostic book-matching helpers. Connectors call their own * search API, then use scoreCandidate() to rank results against the document's - * title/author. Pure functions — unit-tested independently of any network. + * title/author. Pure functions - unit-tested independently of any network. */ /** Fold diacritics, lowercase, drop punctuation, collapse whitespace. */ @@ -39,13 +39,13 @@ export function normalizeAuthor(author: string): string { /** * Derive {title, author} for book matching. The EPUB's own title/author (which * the firmware extracts and sends in the progress `metadata` object) is the real - * signal — this is the primary and expected path. + * signal - this is the primary and expected path. * * Filename is only a last resort for the rare title-less case (a malformed EPUB * whose getTitle() was empty). Note it can't rescue the "no metadata at all" * case: filename ships in the same metadata object as title/author, so if we * lack title we usually lack filename too. We deliberately do NOT guess - * "Title - Author" vs "Author - Title" ordering — we drop the separators and let + * "Title - Author" vs "Author - Title" ordering - we drop the separators and let * the whole string be a fuzzy search query, which search engines handle fine. */ export function extractTitleAuthor(doc: DocumentMeta): { title: string; author: string } | null { @@ -112,7 +112,7 @@ export interface MatchDecision { /** * Rank candidates and decide whether to auto-accept: the top must clear * `threshold` AND beat the runner-up by `margin` (unless the runner-up is the - * same book — same normalized title+author — in which case ambiguity between + * same book - same normalized title+author - in which case ambiguity between * editions is fine and we take the more popular one). */ export function decideMatch( diff --git a/src/connectors/readwise.ts b/src/connectors/readwise.ts index c7aa4e2..b2c0267 100644 --- a/src/connectors/readwise.ts +++ b/src/connectors/readwise.ts @@ -11,10 +11,10 @@ import type { /** * Readwise connector (Tier 1). Official REST API, per-user access token. - * Carries highlights/notes only — NOT reading progress. Bidirectional: + * Carries highlights/notes only - NOT reading progress. Bidirectional: * - fan-out: push CrossInk clippings via POST /api/v2/highlights/ * - fan-in: pull highlights via GET /api/v2/export/ (incl. Kindle, which - * Readwise ingests for us — the "aggregator hop", see docs/design/sync-hub.md). + * Readwise ingests for us - the "aggregator hop", see docs/design/sync-hub.md). * * LIVE-VERIFY GATE: the v2 field names below follow Readwise's documented API * (readwise.io/api_deets) but should be reconfirmed at implementation. Endpoints @@ -51,7 +51,7 @@ async function validate(cred: Credential, http: HttpTransport): Promise [c.id, c])); diff --git a/src/connectors/runner.ts b/src/connectors/runner.ts index d148ee3..d578679 100644 --- a/src/connectors/runner.ts +++ b/src/connectors/runner.ts @@ -62,7 +62,7 @@ export async function processRow(db: DB, row: QueueRow, http: HttpTransport): Pr const connector = getConnector(row.connector_id); const account = getAccount(db, row.user_id, row.connector_id); if (!connector || !account || !account.enabled) { - // Connector gone or disabled — drop permanently. + // Connector gone or disabled - drop permanently. markFailed(db, row, 'connector unavailable or disabled', false); return; } diff --git a/src/connectors/types.ts b/src/connectors/types.ts index 82e54fe..fc066f4 100644 --- a/src/connectors/types.ts +++ b/src/connectors/types.ts @@ -8,7 +8,22 @@ export type Capability = { read: boolean; write: boolean }; /** What data types a connector carries (progress/shelves vs highlights). */ export type DataKind = 'progress' | 'finished' | 'highlight'; -export type CredentialKind = 'token' | 'oauth' | 'cookies'; +export type CredentialKind = 'token' | 'oauth' | 'cookies' | 'kosync' | 'device_code'; + +/** Interactive OAuth device-code link handshake (BookFusion). */ +export interface DeviceLinkStart { + deviceCode: string; + userCode: string; + verificationUri: string; + interval: number; + expiresIn: number; +} +export interface DeviceLinkPoll { + status: 'pending' | 'ok' | 'denied' | 'expired' | 'error'; + credential?: Record; + accountLabel?: string; + error?: string; +} /** Feasibility/trust tier from the design doc. */ export type Tier = 1 | 2 | 3; @@ -34,6 +49,10 @@ export interface OutboundEvent { document: string; /** 0..1 reading fraction (progress/finished events). */ percentage?: number; + /** kosync progress string (xpath/CFI). Present on progress events; used by the kosync mirror. */ + progress?: string; + /** Rich CrossPoint position, forwarded losslessly by the kosync mirror. */ + position?: Record | null; /** unix seconds when this happened on the device/server. */ timestamp: number; /** For highlight events. */ @@ -96,4 +115,9 @@ export interface Connector { /** Push one outbound event. Only called for write-capable connectors. */ push(cred: Credential, match: Match, ev: OutboundEvent, http: HttpTransport): Promise; + + /** Begin an interactive device-code link (OAuth device grant). Optional. */ + beginLink?(http: HttpTransport): Promise; + /** Poll a device-code link until it completes. Optional. */ + pollLink?(deviceCode: string, http: HttpTransport): Promise; } diff --git a/src/crypto/secrets.ts b/src/crypto/secrets.ts index 81deb4e..bbbe02d 100644 --- a/src/crypto/secrets.ts +++ b/src/crypto/secrets.ts @@ -3,7 +3,7 @@ import crypto from 'node:crypto'; /** * Symmetric encryption for connector credentials (third-party tokens and session * cookie bundles) at rest. AES-256-GCM with a random per-record IV; the auth tag - * detects tampering. The key comes from TOKEN_ENC_KEY — if it's unset, all + * detects tampering. The key comes from TOKEN_ENC_KEY - if it's unset, all * connectors are disabled rather than storing secrets in the clear. * * Wire format (base64 of): [1-byte version][12-byte iv][16-byte tag][ciphertext] diff --git a/src/index.ts b/src/index.ts index 9d9af0c..c874680 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,8 +13,8 @@ migrate(db); const app = createApp(db, fromEnv()); -// Connector fan-out queue worker (only meaningful when encryption — hence -// connectors — is configured). +// Connector fan-out queue worker (only meaningful when encryption - hence +// connectors - is configured). const connectorsEnabled = secretsEnabled(); if (connectorsEnabled) { startQueueWorker(db); diff --git a/src/models/position.ts b/src/models/position.ts index 74c8950..f662980 100644 --- a/src/models/position.ts +++ b/src/models/position.ts @@ -1,5 +1,5 @@ /** - * Rich reading position — 1:1 with the firmware's CompactPosition wire struct + * Rich reading position - 1:1 with the firmware's CompactPosition wire struct * (see CrossInk NearbyBookPositionSyncActivity.h). Page fields are layout hints; * pctQ / para / anchor / xpath are the portable parts. */ diff --git a/src/models/stats.ts b/src/models/stats.ts index 7975751..21a23b0 100644 --- a/src/models/stats.ts +++ b/src/models/stats.ts @@ -1,7 +1,7 @@ /** * Reading-stats snapshots mirror CrossInk's GlobalReadingStats / BookReadingStats * (stats_v5). Each device uploads its own snapshot; snapshots are never merged into - * each other — the server aggregates across devices on read, same model as the + * each other - the server aggregates across devices on read, same model as the * firmware's nearby P2P stats sync. */ diff --git a/src/routes/account.ts b/src/routes/account.ts new file mode 100644 index 0000000..181780a --- /dev/null +++ b/src/routes/account.ts @@ -0,0 +1,180 @@ +import crypto from 'node:crypto'; +import { Hono } from 'hono'; +import { deleteCookie } from 'hono/cookie'; +import { withTransaction, type DB } from '../db/db.js'; +import type { Config } from '../config.js'; +import { masterAuth, type AppEnv } from '../auth/middleware.js'; +import { hashKey, verifyKey } from '../auth/password.js'; +import { invalidateAuthCache } from '../auth/middleware.js'; +import { SESSION_COOKIE } from '../auth/session.js'; +import { nowSeconds } from '../models/sync.js'; +import { USERNAME_RE } from './kosync.js'; + +/** Permanently delete a kosync user and every row of its reading data. */ +function deleteKosyncUserData(db: DB, userId: number, username: string): void { + withTransaction(db, () => { + for (const table of [ + 'connector_queue', + 'connector_matches', + 'connector_accounts', + 'stats_device_book', + 'stats_device_global', + 'clippings', + 'bookmarks', + 'documents', + 'progress', + ]) { + db.prepare(`DELETE FROM ${table} WHERE user_id = ?`).run(userId); + } + db.prepare('DELETE FROM users WHERE id = ?').run(userId); + }); + invalidateAuthCache(username); +} + +/** + * Manage the CrossPoint Sync (KOSync) account linked under a master account. + * This is a standard KOReader-compatible sync account: the user picks a username + * and password, exactly like any kosync server, and enters them in their + * reader's KOReader Sync settings. It is where reading data + connectors live. + * A master account owns at most one; it can create a fresh one or link an + * existing (e.g. device-created) one by proving ownership with its password. + * + * The reader sends MD5(password); we store PBKDF2(MD5(password)) in + * users.key_hash, same as any kosync password. The plaintext is never stored. + */ + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +export function accountRoutes(db: DB, config: Config): Hono { + const app = new Hono(); + app.use('*', masterAuth(db)); + + // Status of this account's linked kosync sync identity. + app.get('/kosync', (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT username FROM users WHERE account_id = ?') + .get(account.id) as { username: string } | undefined; + return c.json({ linked: !!row, username: row?.username ?? null }); + }); + + // Create a fresh CrossPoint Sync (KOSync) account with a user-chosen username + // and password, then link it. Standard kosync: the reader uses the same creds. + app.post('/kosync', async (c) => { + const account = c.get('account'); + if (db.prepare('SELECT 1 FROM users WHERE account_id = ?').get(account.id)) { + return c.json({ error: 'A sync account is already linked' }, 409); + } + let username: string | null = null; + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + username = typeof body.username === 'string' ? body.username.trim() : null; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!username || !USERNAME_RE.test(username)) { + return c.json({ error: 'Invalid username' }, 400); + } + if (!password || password.length < 1 || password.length > 256) { + return c.json({ error: 'Invalid password' }, 400); + } + if (db.prepare('SELECT 1 FROM users WHERE username = ?').get(username)) { + return c.json({ error: 'That sync username is taken' }, 409); + } + db.prepare( + 'INSERT INTO users (username, key_hash, account_id, created_at) VALUES (?, ?, ?, ?)' + ).run(username, hashKey(md5(password)), account.id, nowSeconds()); + return c.json({ username }); + }); + + // Link an existing kosync account by proving ownership with its password. + app.put('/kosync', async (c) => { + const account = c.get('account'); + if (db.prepare('SELECT 1 FROM users WHERE account_id = ?').get(account.id)) { + return c.json({ error: 'A sync account is already linked' }, 409); + } + let username: string | null = null; + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + username = typeof body.username === 'string' ? body.username.trim() : null; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!username || !password) { + return c.json({ error: 'Username and password required' }, 400); + } + const row = db + .prepare('SELECT id, key_hash, account_id FROM users WHERE username = ?') + .get(username) as { id: number; key_hash: string; account_id: number | null } | undefined; + // The device sends MD5(password); the web form takes the plain password. + if (!row || !verifyKey(md5(password), row.key_hash)) { + return c.json({ error: 'Invalid sync account credentials' }, 401); + } + if (row.account_id && row.account_id !== account.id) { + return c.json({ error: 'That sync account is linked to another login' }, 409); + } + db.prepare('UPDATE users SET account_id = ? WHERE id = ?').run(account.id, row.id); + return c.json({ linked: true, username }); + }); + + // Detach the kosync account (data stays; it just becomes an orphan again). + app.delete('/kosync', (c) => { + const account = c.get('account'); + db.prepare('UPDATE users SET account_id = NULL WHERE account_id = ?').run(account.id); + return c.json({ linked: false }); + }); + + // Permanently delete the linked kosync account and ALL its reading data. + app.delete('/kosync/data', (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (!row) return c.json({ error: 'No sync account linked' }, 409); + deleteKosyncUserData(db, row.id, row.username); + return c.json({ deleted: true }); + }); + + // Permanently delete the master account, its linked kosync account, and all + // reading data. Signs the browser out. + app.delete('/', (c) => { + const account = c.get('account'); + const kosync = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (kosync) deleteKosyncUserData(db, kosync.id, kosync.username); + db.prepare('DELETE FROM accounts WHERE id = ?').run(account.id); + deleteCookie(c, SESSION_COOKIE, { path: '/' }); + return c.json({ deleted: true }); + }); + + // Change the password of the linked kosync account (reader must be updated). + app.post('/kosync/password', async (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (!row) return c.json({ error: 'No sync account linked' }, 409); + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!password || password.length < 1 || password.length > 256) { + return c.json({ error: 'Invalid password' }, 400); + } + db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run(hashKey(md5(password)), row.id); + invalidateAuthCache(row.username); + return c.json({ username: row.username }); + }); + + return app; +} diff --git a/src/routes/auth.ts b/src/routes/auth.ts index 87abbdf..3b7cefd 100644 --- a/src/routes/auth.ts +++ b/src/routes/auth.ts @@ -16,14 +16,14 @@ import { nowSeconds } from '../models/sync.js'; import { USERNAME_RE } from './kosync.js'; /** - * Token-based web accounts. There are no passwords: signup issues a generated - * token (shown once) that IS the credential. The device sends it as the kosync - * secret (username + token, MD5'd by the client); the web pastes the token to - * establish a session cookie. We store PBKDF2(MD5(token)) in users.key_hash — - * the exact shape device auth already verifies — so one secret works for both. + * Master ("general login") accounts. This is the website identity, kept separate + * from the kosync device credential (see account.ts for linking a kosync sync + * account under a master account). There are no passwords: signup issues a + * master token (shown once) that IS the web credential. We store + * PBKDF2(MD5(token)); the session cookie carries the master account id. * - * Token format: xp1__. The embedded id lets the web log in from - * the token alone (no username needed); security is the random secret + hash. + * Token format: xp1__. The embedded id lets login work from the + * token alone. This token is NOT a kosync secret and never touches a device. */ const TOKEN_RE = /^xp1_(\d+)_[0-9a-f]{32}$/; @@ -32,14 +32,14 @@ function md5(s: string): string { return crypto.createHash('md5').update(s).digest('hex'); } -function makeToken(userId: number): string { - return `xp1_${userId}_${crypto.randomBytes(16).toString('hex')}`; +function masterToken(accountId: number): string { + return `xp1_${accountId}_${crypto.randomBytes(16).toString('hex')}`; } -function setSessionCookie(c: Context, userId: number) { +function setSessionCookie(c: Context, accountId: number) { const proto = c.req.header('x-forwarded-proto'); const secure = proto ? proto.split(',')[0].trim() === 'https' : false; - setCookie(c, SESSION_COOKIE, signSession(userId), { + setCookie(c, SESSION_COOKIE, signSession(accountId), { httpOnly: true, sameSite: 'Lax', secure, @@ -51,55 +51,54 @@ function setSessionCookie(c: Context, userId: number) { export function authRoutes(db: DB, config: Config): Hono { const app = new Hono(); - // Create an account: pick a username, receive a token (shown once). + // Create a master account: pick a handle, receive a login token (shown once). app.post('/signup', rateLimiter(config.authRateLimitPerMinute), async (c) => { if (config.registrationDisabled) { return c.json({ error: 'Registration is disabled' }, 403); } - let username: string | null = null; + let handle: string | null = null; try { const body = (await c.req.json()) as Record; - username = typeof body.username === 'string' ? body.username.trim() : null; + handle = typeof body.handle === 'string' ? body.handle.trim() : null; } catch { - /* fall through to validation error */ + /* validation below */ } - if (!username || !USERNAME_RE.test(username)) { - return c.json({ error: 'Invalid username' }, 400); + if (!handle || !USERNAME_RE.test(handle)) { + return c.json({ error: 'Invalid handle' }, 400); } - if (db.prepare('SELECT 1 FROM users WHERE username = ?').get(username)) { - return c.json({ error: 'Username is already registered' }, 409); + if (db.prepare('SELECT 1 FROM accounts WHERE handle = ?').get(handle)) { + return c.json({ error: 'Handle is already taken' }, 409); } - // Insert to get the id, then bake it into the token and store its hash. - const token = crypto.randomBytes(16).toString('hex'); + const secret = crypto.randomBytes(16).toString('hex'); const info = db - .prepare('INSERT INTO users (username, key_hash, created_at) VALUES (?, ?, ?)') - .run(username, '', nowSeconds()); - const userId = Number(info.lastInsertRowid); - const fullToken = `xp1_${userId}_${token}`; - db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run(hashKey(md5(fullToken)), userId); - setSessionCookie(c, userId); - return c.json({ username, token: fullToken }); + .prepare('INSERT INTO accounts (handle, token_hash, created_at) VALUES (?, ?, ?)') + .run(handle, '', nowSeconds()); + const accountId = Number(info.lastInsertRowid); + const token = `xp1_${accountId}_${secret}`; + db.prepare('UPDATE accounts SET token_hash = ? WHERE id = ?').run(hashKey(md5(token)), accountId); + setSessionCookie(c, accountId); + return c.json({ handle, token }); }); - // Web login: paste the token, get a session cookie. + // Web login: paste the master token, get a session cookie. app.post('/login', rateLimiter(config.authRateLimitPerMinute), async (c) => { let token: string | null = null; try { const body = (await c.req.json()) as Record; token = typeof body.token === 'string' ? body.token.trim() : null; } catch { - /* fall through */ + /* validation below */ } const parsed = token?.match(TOKEN_RE); if (!token || !parsed) return c.json({ error: 'Invalid token' }, 401); - const userId = Number(parsed[1]); + const accountId = Number(parsed[1]); const row = db - .prepare('SELECT key_hash FROM users WHERE id = ?') - .get(userId) as { key_hash: string } | undefined; - if (!row || !verifyKey(md5(token), row.key_hash)) { + .prepare('SELECT token_hash FROM accounts WHERE id = ?') + .get(accountId) as { token_hash: string } | undefined; + if (!row || !verifyKey(md5(token), row.token_hash)) { return c.json({ error: 'Invalid token' }, 401); } - setSessionCookie(c, userId); + setSessionCookie(c, accountId); return c.json({ ok: true }); }); @@ -112,25 +111,25 @@ export function authRoutes(db: DB, config: Config): Hono { const session = verifySession(getCookie(c, SESSION_COOKIE)); if (!session) return c.json({ error: 'Not signed in' }, 401); const row = db - .prepare('SELECT username FROM users WHERE id = ?') - .get(session.uid) as { username: string } | undefined; + .prepare('SELECT handle FROM accounts WHERE id = ?') + .get(session.uid) as { handle: string } | undefined; if (!row) return c.json({ error: 'Not signed in' }, 401); - return c.json({ username: row.username }); + return c.json({ handle: row.handle }); }); - // Rotate the token (revokes the old one — the device must be updated). - // Requires an active web session. + // Rotate the master login token (revokes the old one). app.post('/token/rotate', (c) => { const session = verifySession(getCookie(c, SESSION_COOKIE)); if (!session) return c.json({ error: 'Not signed in' }, 401); - const exists = db.prepare('SELECT 1 FROM users WHERE id = ?').get(session.uid); - if (!exists) return c.json({ error: 'Not signed in' }, 401); - const fullToken = makeToken(session.uid); - db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run( - hashKey(md5(fullToken)), + if (!db.prepare('SELECT 1 FROM accounts WHERE id = ?').get(session.uid)) { + return c.json({ error: 'Not signed in' }, 401); + } + const token = masterToken(session.uid); + db.prepare('UPDATE accounts SET token_hash = ? WHERE id = ?').run( + hashKey(md5(token)), session.uid ); - return c.json({ token: fullToken }); + return c.json({ token }); }); return app; diff --git a/src/routes/kosync.ts b/src/routes/kosync.ts index 869fe69..061f209 100644 --- a/src/routes/kosync.ts +++ b/src/routes/kosync.ts @@ -16,7 +16,7 @@ import { fanOutProgress } from '../connectors/fanout.js'; export const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; export function isValidDocument(v: unknown): v is string { - // KOReader sends a 32-hex MD5, but the key is opaque — stay lenient. + // KOReader sends a 32-hex MD5, but the key is opaque - stay lenient. return typeof v === 'string' && /^[A-Za-z0-9._-]{1,64}$/.test(v); } @@ -189,7 +189,7 @@ export function kosyncRoutes(db: DB, config: Config): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); - fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt, parsed.record.progress, parsed.record.position); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); diff --git a/src/routes/v1/connectors.ts b/src/routes/v1/connectors.ts index ab32603..b4cd6d7 100644 --- a/src/routes/v1/connectors.ts +++ b/src/routes/v1/connectors.ts @@ -81,6 +81,56 @@ export function connectorRoutes(db: DB, transport: HttpTransport = fetchTranspor return c.json({ id: conn.id, linked: true, account: result.accountLabel ?? null }); }); + // Begin an interactive device-code link (BookFusion). Returns the user code + + // verification URL for the browser to show, and the device code to poll with. + app.post('/connectors/:id/link/begin', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!conn.beginLink) return c.json({ code: 2003, message: 'Connector has no device link' }, 400); + if (!secretsEnabled()) { + return c.json({ code: 2003, message: 'Server has no TOKEN_ENC_KEY; connector storage disabled' }, 403); + } + try { + const start = await conn.beginLink(transport); + return c.json({ + device_code: start.deviceCode, + user_code: start.userCode, + verification_uri: start.verificationUri, + interval: start.interval, + expires_in: start.expiresIn, + }); + } catch (err) { + return c.json({ code: 2003, message: err instanceof Error ? err.message : 'Link failed' }, 502); + } + }); + + // Poll a device-code link; on success, store the credential and link. + app.post('/connectors/:id/link/poll', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!conn.pollLink) return c.json({ code: 2003, message: 'Connector has no device link' }, 400); + let deviceCode: unknown; + try { + deviceCode = ((await c.req.json()) as Record).device_code; + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + if (typeof deviceCode !== 'string') return kosyncError(c, 403, 2003, 'Invalid request'); + try { + const result = await conn.pollLink(deviceCode, transport); + if (result.status === 'ok' && result.credential) { + const user = c.get('user'); + // Confirm the freshly minted credential works, then store it. + const v = await conn.validate(result.credential, transport); + upsertAccount(db, user.id, conn.id, result.credential, v.accountLabel ?? result.accountLabel ?? null); + return c.json({ status: 'ok', linked: true }); + } + return c.json({ status: result.status, error: result.error ?? null }); + } catch (err) { + return c.json({ status: 'error', error: err instanceof Error ? err.message : 'poll failed' }, 502); + } + }); + // Unlink and wipe queued work + matches. app.delete('/connectors/:id', (c) => { const conn = getConnector(c.req.param('id')); @@ -115,7 +165,7 @@ export function connectorRoutes(db: DB, transport: HttpTransport = fetchTranspor }); }); - // Manually set/override a match (sticky — never auto-recomputed). + // Manually set/override a match (sticky - never auto-recomputed). app.put('/connectors/:id/matches/:document', async (c) => { const conn = getConnector(c.req.param('id')); if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); @@ -133,7 +183,7 @@ export function connectorRoutes(db: DB, transport: HttpTransport = fetchTranspor const externalId = o.external_id; const user = c.get('user'); if (externalId === null) { - // Explicit "no match" override — stop trying to sync this document. + // Explicit "no match" override - stop trying to sync this document. saveMatch(db, user.id, conn.id, document, null, 'manual'); return c.json({ document, external_id: null, source: 'manual' }); } @@ -155,7 +205,7 @@ export function connectorRoutes(db: DB, transport: HttpTransport = fetchTranspor return c.json({ document, external_id: externalId, source: 'manual' }); }); - // Force (re)matching of a document now — useful for testing and the review UI. + // Force (re)matching of a document now - useful for testing and the review UI. app.post('/connectors/:id/rematch/:document', async (c) => { const conn = getConnector(c.req.param('id')); if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); diff --git a/src/routes/v1/progress.ts b/src/routes/v1/progress.ts index 73e14a7..9a692a7 100644 --- a/src/routes/v1/progress.ts +++ b/src/routes/v1/progress.ts @@ -20,12 +20,12 @@ export function progressRoutes(db: DB): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); - fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt, parsed.record.progress, parsed.record.position); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); // List every synced document with its newest progress (joined with any known - // metadata) — lets clients and UIs discover documents without knowing hashes. + // metadata) - lets clients and UIs discover documents without knowing hashes. app.get('/progress', (c) => { const user = c.get('user'); const limitRaw = Number(c.req.query('limit') ?? 100); diff --git a/src/routes/web.ts b/src/routes/web.ts index 501f69e..5515095 100644 --- a/src/routes/web.ts +++ b/src/routes/web.ts @@ -78,6 +78,8 @@ const STYLE = ` button.primary:hover { background:var(--brand-600); } button.ghost { background:#fff; color:var(--stone-700); box-shadow:0 0 0 1px var(--stone-200); } button.ghost:hover { background:var(--stone-50); } + button.danger { background:#fff; color:#b91c1c; box-shadow:0 0 0 1px #f0cccc; } + button.danger:hover { background:#fef2f2; } button.copied { color:var(--brand-700); box-shadow:0 0 0 1px var(--brand-100); background:var(--brand-50); opacity:1; } button:disabled { opacity:.5; cursor:default; } button.copied:disabled { opacity:1; } @@ -131,26 +133,26 @@ const LANDING = shell(

Create account

- +

Sign in

- +
@@ -166,7 +168,7 @@ async function post(url, body) { } $('signup').onclick = async () => { $('suErr').textContent = ''; - const { ok, data } = await post('/auth/signup', { username: $('su').value.trim() }); + const { ok, data } = await post('/auth/signup', { handle: $('su').value.trim() }); if (!ok) { $('suErr').textContent = data.error || 'Something went wrong'; return; } $('tokenVal').textContent = data.token; $('tokenBox').hidden = false; @@ -197,6 +199,8 @@ $('login').onclick = async () => { ` ); +const SECTION = 'font-size:13px;text-transform:uppercase;letter-spacing:0.05em;color:var(--stone-500);margin:32px 0 12px;'; + const ACCOUNT = shell( 'Account', `
@@ -204,13 +208,19 @@ const ACCOUNT = shell(

Signed in as …

-

Linked services

+

CrossPoint Sync (KOSync)

+

Loading…

+ +

Linked services

Loading…

-

Your sync token

+

Website login

-

Use this as the password on your reader (KOReader / CrossPoint sync settings), with your username. Rotating it signs your reader out until you enter the new one.

- +

The token you use to sign into this website. Rotating it signs you out on the web and does not affect your reader.

+
+ + +