diff --git a/Dockerfile b/Dockerfile index 9284e4b..43c0ffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ WORKDIR /app COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/dist ./dist COPY migrations ./migrations +COPY assets ./assets COPY package.json ./ RUN mkdir -p /data && chown node:node /data USER node diff --git a/README.md b/README.md index f2b957e..5d8c2d0 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ DATABASE_PATH=./data/crosspoint.db PORT=8080 node dist/index.js | `DATABASE_PATH` | `/data/crosspoint.db` | SQLite file (parent dirs auto-created) | | `REGISTRATION_DISABLED` | `false` | Set `true` to lock down a private instance | | `AUTH_RATE_LIMIT_PER_MINUTE` | `30` | Per-IP limit on registration (0 disables) | +| `TOKEN_ENC_KEY` | _(unset)_ | Enables connectors (Hardcover/Readwise sync). 64 hex chars, a base64 32-byte key, or a ≥32-char passphrase. Encrypts stored connector credentials at rest; unset = connectors disabled. | ## Point your reader at it diff --git a/assets/favicon.png b/assets/favicon.png new file mode 100644 index 0000000..42f6441 Binary files /dev/null and b/assets/favicon.png differ diff --git a/assets/logo.png b/assets/logo.png new file mode 100644 index 0000000..6d8bed7 Binary files /dev/null and b/assets/logo.png differ diff --git a/docs/API.md b/docs/API.md index 186928a..2a201ca 100644 --- a/docs/API.md +++ b/docs/API.md @@ -374,6 +374,62 @@ weighted by `pace_n`, `start_date` = earliest non-zero, `finished_date` = latest `GET /api/v1/documents` lists them. Most clients don't need this endpoint — progress-PUT `metadata` capture populates the same table. +### Connectors (master sync hub) + +Pair external services (Hardcover, Readwise, …) to the account; reading activity fans out to them +server-side. See docs/design/sync-hub.md. All under `/api/v1`, same auth headers. Requires the +server to have `TOKEN_ENC_KEY` set (credentials are encrypted at rest) — otherwise these endpoints +report `encryption: "disabled"` and linking returns 403. + +#### GET /api/v1/connectors + +Lists available connectors and this account's link status. + +```json +{ + "encryption": "enabled", + "connectors": [ + {"id": "hardcover", "name": "Hardcover", "tier": 1, "experimental": false, + "carries": ["progress", "finished"], "capabilities": {"read": false, "write": true}, + "credential_kind": "token", "linked": true, "status": "ok", "account": "julia", + "queue": {"pending": 0, "dead": 0}}, + {"id": "readwise", "name": "Readwise", "tier": 1, "experimental": false, + "carries": ["highlight"], "capabilities": {"read": true, "write": true}, + "credential_kind": "token", "linked": false, "status": null, "account": null} + ] +} +``` + +#### PUT /api/v1/connectors/{id} + +Link/re-link by validating and storing a credential. Body: `{"credential": { ... }}` — shape is +connector-specific (`{"token": "..."}` for Hardcover and Readwise). The server validates against the +service before storing; returns `400` if rejected. `{"id": "hardcover", "linked": true, +"account": "julia"}` on success. + +#### DELETE /api/v1/connectors/{id} + +Unlink; wipes the stored credential, all matches, and queued work. + +#### GET /api/v1/connectors/{id}/matches + +Lists resolved book matches (for a review UI): `{"connector": "hardcover", "matches": [{"document", +"external_id", "confidence", "source": "auto|manual|none", "query_used", "updated_at"}]}`. + +#### PUT /api/v1/connectors/{id}/matches/{document} + +Manually set a match (sticky — never auto-recomputed). Body `{"external_id": "42"}`, or +`{"external_id": null}` to mark "never sync this document". + +#### POST /api/v1/connectors/{id}/rematch/{document} + +Force (re)matching now; returns the resolved match or null. Preserves manual overrides. + +**Matching** is server-side from the document's title/author (the EPUB metadata the firmware sends — +so connectors need "Send Metadata" on). **Fan-out** is automatic: a progress PUT enqueues a +progress/finished event to write-connectors that carry it; a clippings PUT enqueues highlight events +to highlight-connectors (Readwise). A background worker delivers them with retry/backoff. + ### GET /healthz Unauthenticated. `{"status": "ok", "version": "0.1.0"}`. diff --git a/docs/BUILD_STATUS.md b/docs/BUILD_STATUS.md new file mode 100644 index 0000000..cf8a370 --- /dev/null +++ b/docs/BUILD_STATUS.md @@ -0,0 +1,86 @@ +# Build status — connector hub (overnight session) + +Snapshot of the master-sync-hub build. **Nothing committed or pushed.** Everything below is in the +working tree of `crosspoint-sync` for your review. + +## What's done and tested + +The full connector framework from `docs/design/sync-hub.md`, plus both Tier-1 connectors. +`npm test` = **74 passing** (was 40); `npx tsc --noEmit` clean; `npm run build` clean. + +### Framework +- **Encrypted secret vault** (`src/crypto/secrets.ts`) — AES-256-GCM, key from `TOKEN_ENC_KEY` + (hex / base64 / passphrase). Connectors disabled when unset; never stores plaintext. Tested + (roundtrip, tamper-detection, key formats, disabled mode). +- **Schema** (`migrations/0003_connectors.sql`) — `connector_accounts`, `connector_matches`, + `connector_queue`, keyed by `connector_id`. +- **Connector interface + registry** (`src/connectors/types.ts`, `registry.ts`) — one module per + service; `HttpTransport` is injectable so everything is testable without network. +- **Matcher** (`src/connectors/matching.ts`) — server-side title/author matching: normalize, + strip subtitles/series, `Last, First` handling, Jaccard scoring, auto-accept only on a clear + winner (same-book editions OK, title collisions rejected). Filename is a last resort only. + Heavily unit-tested. +- **Coalescing retry queue** (`src/connectors/queue.ts`) — one pending row per + (user, connector, document, coalesceKey); progress collapses to latest, highlights keyed per + clipping; exponential backoff, dead-letter after 8 tries. Tested. +- **Runner/worker** (`src/connectors/runner.ts`) — resolves match (cached/manual-sticky/auto), + pushes, handles reauth/backoff; `startQueueWorker` drains every 15s (started in `index.ts` only + when `TOKEN_ENC_KEY` is set). +- **Fan-out** (`src/connectors/fanout.ts`) — progress PUT → progress/finished events; clippings + PUT → highlight events. Wired into both kosync and v1 progress routes and the clippings route. + Best-effort, never blocks the request. +- **Management API** (`src/routes/v1/connectors.ts`) — list / link / unlink / list-matches / + manual-match / rematch. Documented in `docs/API.md`. Endpoint + fan-out tests in + `test/connectors.test.ts` (fake transport). + +### Connectors +- **Hardcover** (`src/connectors/hardcover.ts`, Tier 1) — token validate, search-based match, + status mutation (reading/read). Write-only, carries progress+finished. +- **Readwise** (`src/connectors/readwise.ts`, Tier 1) — token validate, highlight push (fan-out), + and `exportHighlights()` for the fan-in "aggregator hop" (Kindle highlights via Readwise). + Carries highlights. + +## ⚠️ Live-verify gates before enabling in production + +Both connectors are built against **documented** API shapes but not verified against live accounts +(I have no tokens, and shouldn't hit third-party APIs from here). Search the code for `GATE`. + +- **Hardcover** (beta API — highest risk): confirm against the live GraphQL explorer + (hardcover.app/account/api): the `me` query shape, `search` result shape (see `extractSearchHits`, + which defensively handles a few shapes), the `insert_user_book` mutation name/args, and the + **status ids** (`STATUS_READING`/`STATUS_READ` are guesses). Wire a recorded-fixture test once + confirmed so schema drift breaks CI. +- **Readwise** (stable public API — lower risk): confirm `POST /api/v2/highlights/` field names and + the `GET /api/v2/export/` cursor field (`nextPageCursor`). Mind the ~20 req/min limit on + create/export. + +## Not built (deliberately deferred) + +- **Fan-in wiring** — `exportHighlights()` exists but isn't hooked into a canonical-clippings + importer or a poll loop yet. That's the next chunk if you want Kindle-via-Readwise highlights + landing in the clippings store / on-device. +- **Tier 2/3 connectors** (Goodreads/StoryGraph cookie-replay, Kindle) — framework supports them + (`credentialKind: 'cookies'`, `experimental` flag) but none implemented; they need the CSRF + handshake + live capture work described in sync-hub.md, behind an experimental opt-in. +- **Web UI** — token paste / OAuth / match-review screen. The API is UI-ready; the UI is the + natural next project and the real prerequisite for non-technical pairing. +- **Bidirectional/fan-in merge rules** beyond the design notes. + +## To run locally with connectors on + +```sh +export TOKEN_ENC_KEY=$(openssl rand -hex 32) +DATABASE_PATH=./data/dev.db npm run dev +# GET /api/v1/connectors shows encryption: "enabled" and both connectors unlinked +``` + +Railway note: to enable connectors in prod, set `TOKEN_ENC_KEY` as a service variable (generate +once, keep it stable — rotating it invalidates all stored connector credentials). + +## Suggested review order + +1. `docs/design/sync-hub.md` (the plan) → `src/connectors/types.ts` (the shape). +2. `matching.ts` + `test/matching.test.ts` (the only tricky pure logic). +3. `runner.ts` + `queue.ts` + `test/queue.test.ts` (delivery semantics). +4. `hardcover.ts` / `readwise.ts` (check the GATEs against live docs). +5. `test/connectors.test.ts` (end-to-end link → sync → fan-out with a fake API). diff --git a/docs/design/hardcover-sync.md b/docs/design/hardcover-sync.md new file mode 100644 index 0000000..c5eb0d8 --- /dev/null +++ b/docs/design/hardcover-sync.md @@ -0,0 +1,175 @@ +# Design: Hardcover Sync Connector + +Status: **draft / not implemented** + +Forward reading activity from crosspoint-sync to [Hardcover](https://hardcover.app) so a user's +shelf reflects what they read on their e-reader — automatically. The connector lives entirely +server-side: devices keep speaking plain kosync and never talk to Hardcover. + +## Goals + +- When a user reads on any synced device, their Hardcover profile updates: book moves to + "Currently Reading" on first progress, progress percentage stays current, and the book flips to + "Read" (with a finish date) on completion. +- Zero firmware changes. Works for CrossPoint, CrossInk, and stock KOReader (once it ships the + metadata PR) alike. +- Hardcover being slow, down, or rate-limiting must never affect device sync. + +Non-goals (v1): syncing clippings/notes to Hardcover, importing Hardcover state back to devices, +StoryGraph (no public API), ratings/reviews. + +## Architecture + +``` +device --kosync PUT--> crosspoint-sync --enqueue--> forwarder --GraphQL--> Hardcover + | ^ + +-- matcher (title/author search, once per document) +``` + +Three pieces, all in this codebase: + +1. **Token link** — user attaches their Hardcover API token to their account. +2. **Matcher** — resolves our opaque document hash to a Hardcover book, using the + title/author/filename metadata we already capture. Runs once per `(user, document)`, result + cached; re-runs when metadata improves or the user overrides. +3. **Forwarder** — turns progress events into Hardcover GraphQL mutations, with a retry queue. + +### Why matching is server-side (decision) + +Title/author already arrive via the progress-PUT `metadata` object, so every existing client feeds +the matcher with no firmware release. Hardcover's search API (Typesense-backed) handles the fuzzy +matching; we only rank/accept results. Server-side logic can be improved and re-run against +historical documents at any time. Firmware-extracted ISBN was considered and rejected for the +critical path: it needs a firmware release to exist, and `dc:identifier` is missing or wrong in +enough real-world EPUBs (DRM-stripped, self-published, Calibre-converted) that we'd need the +fuzzy path anyway. ISBN remains a possible future confidence booster. + +## Hardcover API notes + +- Endpoint: `https://api.hardcover.app/v1/graphql`, header `Authorization: Bearer `. +- Tokens are user-generated (hardcover.app/account/api) and currently expire ~yearly; the API is + officially beta. Treat every request as fallible and every schema detail as subject to change. +- **Implementation gate:** before building the forwarder, verify against the live schema (the API + has a GraphQL explorer): search query shape, `user_books` status ids + (want-to-read / currently-reading / read), and the exact mutations for inserting/updating a + user book and its progress (book-level vs edition-level, pages vs percentage). Encode those in + one `src/connectors/hardcover.ts` module with a recorded-fixture test so schema drift breaks CI, + not production. +- Rate limits are modest (beta). The forwarder must coalesce: at most one progress mutation per + (user, book) per N minutes (default 15), always keeping only the latest value. Reading sessions + on e-ink produce sparse syncs anyway (sync is user-initiated over WiFi), so this is cheap. + +## Schema additions + +```sql +CREATE TABLE hardcover_accounts ( + user_id INTEGER PRIMARY KEY REFERENCES users(id), + token_enc TEXT NOT NULL, -- encrypted at rest (see Security) + enabled INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE hardcover_matches ( + user_id INTEGER NOT NULL REFERENCES users(id), + document TEXT NOT NULL, + book_id INTEGER, -- Hardcover book id (NULL = unmatched) + edition_id INTEGER, -- optional, when confidently known + confidence REAL NOT NULL DEFAULT 0, + source TEXT NOT NULL, -- 'auto' | 'manual' | 'none' + query_used TEXT, -- what we searched, for debugging/review UI + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, document) +); + +CREATE TABLE hardcover_queue ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + document TEXT NOT NULL, + kind TEXT NOT NULL, -- 'progress' | 'finished' + payload TEXT NOT NULL, -- JSON: percentage, timestamp, finished_date... + attempts INTEGER NOT NULL DEFAULT 0, + next_try_at INTEGER NOT NULL, + UNIQUE (user_id, document, kind) -- coalesce: newest payload replaces +); +``` + +## API additions (same x-auth headers) + +- `PUT /api/v1/connectors/hardcover` — `{token: "..."}`; server validates it with a cheap `me` + query before storing. `DELETE` unlinks and wipes the queue. +- `GET /api/v1/connectors/hardcover` — `{enabled, linked_at, matched: n, unmatched: n}`. +- `GET /api/v1/connectors/hardcover/matches` + `PUT .../matches/:document` — list and manually + set/override matches. Powers the future web-UI "review matches" screen; usable via curl until + then. Manual matches are sticky (`source = 'manual'`, never auto re-matched). + +Token entry realistically requires a browser, not an e-ink keyboard — this lands with (or just +before) the first web UI. The endpoints don't depend on the UI, though. + +## Matching algorithm + +Input: the EPUB's **`documents.title` + `documents.author`** — the real signal, which the firmware +already extracts and sends in the progress `metadata` object. This is the primary/expected path. +Filename is only a last resort for a title-less EPUB, and we don't guess `Title - Author` vs +`Author - Title` order (drop separators, use the whole string as a fuzzy query). Filename can't +rescue the no-metadata case — title/author and filename ship in the *same* metadata object — so +**matching effectively requires "Send Metadata" on**, which is the reason to default that toggle on +for connector users. + +1. Normalize: strip subtitle after `:`, series suffixes like `(Book 2)`, diacritics folded, + `Last, First` → `First Last`. +2. Query Hardcover search with `title author`. +3. Score the top results: normalized-title similarity + author-name overlap. Accept the top hit + when it clears a threshold **and** clearly beats the runner-up (title collisions like + "Circe" resolve on author; same-title-same-author editions are all correct at book level, so + ambiguity between them is acceptable — take the most popular edition). +4. Store the result either way (`source: 'auto'` or `'none'` with the query kept for review). + +Re-match triggers: metadata for the document changes (e.g. the user flips Send Metadata on after +the fact), or a manual re-match request. Never re-match over `source = 'manual'`. + +Documents with no metadata at all simply stay unmatched — which is also the user-facing nudge to +enable the Send Metadata toggle. + +## Forwarding rules + +On progress PUT for a linked user with a matched document, upsert into the queue: + +- percentage > 0 → `progress` event: ensure user_book exists with status currently-reading, + update progress percentage. First event for a book is what shelves it. +- completion → `finished` event: status read + finish date. Completion = percentage ≥ 0.98, or — + once CrossInk stats sync ships — `isCompleted` / non-zero `finished_date` from + `stats_device_book` (the stronger signal wins; stats-based finish dates are real dates, the + percentage heuristic uses the sync timestamp). + +A small worker drains the queue (setInterval in the Node process — no external infra), with +exponential backoff on failure and a dead-letter state after ~a week of retries. `401` from +Hardcover (expired token) disables the link and surfaces in the status endpoint rather than +retrying forever. + +Kosync GET/downstream sync is never blocked or delayed by any of this. + +## Security + +- Hardcover tokens are bearer credentials to a third-party account: encrypt at rest + (AES-256-GCM with a key from a `TOKEN_ENC_KEY` env var), never log them, redact in errors. + Self-hosters without `TOKEN_ENC_KEY` set get the connector disabled, not plaintext storage. +- Outbound requests go only to the Hardcover endpoint; no user-controlled URLs (no SSRF surface). + +## Rollout + +1. Schema + token link + matcher, with `GET .../matches` for inspection. Verify match quality on + real libraries via curl before any forwarding exists. +2. Forwarder for progress + finished, behind per-user `enabled`. +3. Web UI: token entry + review-matches screen. +4. Later: ISBN confidence boost, richer status mapping (owned/DNF), maybe Bookwyrm/Calibre-Web + via the same connector pattern. + +## Open questions + +- Exact Hardcover mutation set and status ids (resolve at the implementation gate above). +- Should percentage-completion threshold be user-configurable? (Default 0.98; back-matter skews + short books.) +- Multi-device: progress forwarding uses the newest-across-devices row (same rule as kosync GET), + so no per-device fan-out — confirm that's the desired UX for people who read the same book on + two devices at different points. diff --git a/docs/design/sync-hub.md b/docs/design/sync-hub.md new file mode 100644 index 0000000..b0a3203 --- /dev/null +++ b/docs/design/sync-hub.md @@ -0,0 +1,258 @@ +# Design: Sync Hub & Connector Framework + +Status: **draft / not implemented** — architectural direction, not a commitment to ship every +connector below. Read the feasibility tiers before scoping anything. + +## Idea + +A crosspoint-sync account becomes a **master sync identity**. Reading state (progress, finished +status, and later ratings/clippings) is captured once — from any device over kosync — and +fanned out to whichever external services the user has *paired*: Hardcover, Goodreads, StoryGraph, +Kindle/Whispersync, Audible, etc. All pairing and forwarding happens server-side, so there are +**no firmware changes** for any of it. + +[hardcover-sync.md](hardcover-sync.md) is the reference connector; this doc generalizes that +pattern to N connectors and defines what's actually buildable. + +## The hub, conceptually + +``` + crosspoint-sync account (master identity) + | + +------------------+------------+------------+------------------+ + | | | | + device sync Hardcover Goodreads Kindle / Audible + (kosync, canonical) connector connector connector + | | | | + progress/stats <--- sync graph: canonical store + per-connector adapters ---> +``` + +- **Canonical store** = the tables we already have (`progress`, `documents`, stats, bookmarks, + clippings). This is the source of truth and the lowest common denominator. +- **Connectors** are adapters. Each declares: which credential type it needs, whether it's + read/write/bidirectional, how it matches our document hash to its own book identity, and how it + maps our state to its state. +- **Fan-out** is triggered by a change to the canonical store (a progress PUT). **Fan-in** + (bidirectional connectors pushing external changes back into the canonical store) is a poll or + webhook per connector — only some connectors can do it. + +## Connector interface (shared shape) + +Every connector, regardless of service, is one module implementing: + +```ts +interface Connector { + id: string; // 'hardcover' | 'goodreads' | ... + capabilities: { read: boolean; write: boolean }; // fan-in / fan-out support + credentialKind: 'oauth' | 'token' | 'cookies'; // never 'password' — see auth models + validate(cred): Promise<{ ok: boolean; account?: string }>; + match(doc: DocumentMeta): Promise; // hash -> external book id + pushProgress(cred, match, ev): Promise; // fan-out (write connectors) + pullChanges?(cred, since): Promise; // fan-in (read connectors) +} +``` + +This reuses the Hardcover design's three-part structure (token vault, match table, retry queue) +for all connectors. Adding a service = adding one module + one row in a connector registry; the +hub, queue, matcher-runner, and web UI are written once. Matching is **always server-side**, using +the title/author/filename metadata the devices already send (see hardcover-sync.md — the same +"why server-side" reasoning applies universally). + +Shared tables generalize the Hardcover ones: `connector_accounts(user_id, connector_id, +cred_enc, enabled, ...)`, `connector_matches(user_id, connector_id, document, external_id, +confidence, source, ...)`, `connector_queue(...)`. One schema, keyed by `connector_id`. + +## Auth models — the key insight: cookie-replay ≠ storing passwords + +The connector landscape is uneven, but the dividing line is **auth mechanism**, not "has an API." +Three models, best to worst posture: + +1. **Scoped token / OAuth (Tier 1).** Service issues a per-user token with the user's consent. + Revocable, sometimes scopeable. Best case. (Hardcover.) +2. **Cookie-replay (Tier 2).** User is already logged in on the service in their browser; they + paste their session cookies (or a browser extension harvests just those cookies). The connector + replays them against the service's own **web endpoints**. **The server never sees the + password.** Because the cookie is captured *post-login*, this also sidesteps 2FA/CAPTCHA + entirely. This is the standard modern pattern — it's how Readwise's Kindle sync works + ("Readwise couldn't access your Amazon password even if we wanted to"). The tradeoff isn't + credential exposure — it's that session cookies are unscopeable full-session secrets, expire and + need re-pasting, and depend on undocumented web endpoints that can change. +3. **Stored password / headless login (avoid).** Server holds the actual username+password and + drives a login. Highest risk; breaks on 2FA. **We do not build this for any connector.** + +Cookie-replay is what makes Goodreads/StoryGraph/Kindle viable at all now — and viable *without* +holding anyone's password. It's still ToS-gray and brittle, so those connectors ship behind an +explicit "experimental, may break, unofficial" opt-in — but "we store your password" was never +the actual requirement, and I was wrong to frame it that way earlier. + +### Prefer the aggregator hop over direct scraping + +When a **sanctioned aggregator already ingests a hard target**, route through it instead of +scraping the target ourselves. The prime example: **Readwise** already pulls Kindle highlights via +its own browser extension (the ToS-gray Amazon work is *their* responsibility, done with the user's +explicit install), and exposes them through an official token API. So for Kindle *highlights*, a +Readwise connector (Tier 1, no scraping) beats a direct Kindle cookie-replay connector (Tier 3, +TLS-fingerprint fight). Always ask "is there a legit API that already has this data?" before +building a scraper. (This doesn't cover Kindle reading-*progress* — Readwise doesn't have it — so +the direct-Kindle spike still stands for progress specifically.) + +## Feasibility tiers — READ THIS BEFORE SCOPING + +### Tier 1 — Sanctioned public API. Buildable, durable. + +- **Hardcover** — public GraphQL API (beta), per-user bearer tokens. Write + partial read. + Carries *progress + shelves + rating*. See hardcover-sync.md. **Ship first.** +- **Readwise** — official REST API, per-user access token (`Authorization: Token `, issued at + readwise.io/access_token). Carries *highlights/notes only* — no reading progress or shelves. + **Bidirectional and the safest connector we have**, because it's a fully sanctioned token API + (no cookies, no scraping, user-revocable). Two high-value uses: + - **Fan-out:** push CrossInk clippings into Readwise (`POST /api/v2/highlights/`, batched) — from + there they flow to the user's whole highlight ecosystem (Notion, Obsidian, Readwise reviews). + This is the cleanest way to make on-device highlights useful anywhere. + - **Fan-in / "Readwise as the hop":** pull highlights *out* of Readwise + (`GET /api/v2/export/?updatedAfter=`) into our canonical clippings store. Since Readwise's own + browser extension already ingests **Kindle** (and Apple Books, Instapaper, …) highlights the + sanctioned way, this gets us Kindle highlights **without us ever scraping Amazon** — we let + Readwise do the Amazon work and read from their clean API. This is the safer alternative to a + direct Kindle connector for the *highlights* use case. + + Caveats: highlights-only (does **not** solve Kindle reading-*progress* sync — that's still the + Tier-3 spike below), requires a paid Readwise subscription, and create/export endpoints are + rate-limited (~20 req/min — batch accordingly). Book identity in Readwise is title/author, which + matches our metadata model. Verify the exact v2 schema at implementation (same gate as Hardcover). + +### Tier 2 — No official API, but cookie-replay works. Experimental, no passwords stored. + +- **Goodreads** — API keys dead since **Dec 2020**. The well-known Calibre "Goodreads Sync" plugin + still works only because it ships the author's own *grandfathered* OAuth key, shared across its + whole userbase — **we can't and shouldn't reuse that** (a new app can't register a key; borrowing + theirs invites revocation + ToS violation). The viable path for us is **cookie-replay against the + Goodreads web endpoints** (`_session_id2` session cookie + Rails `authenticity_token` CSRF + scraped from page HTML). This can *write*: shelve, rate, set read date, and update reading + progress. See "Goodreads write-path" below. +- **StoryGraph** — no public API, but cookie-replay is proven: the `storygraph.koplugin` KOReader + plugin already writes progress %, status, and auto-marks Read using two cookies + (`_story_graph_session` + `remember_user_token`) + CSRF. **Note the competitive context: Kobo + shipped *native* StoryGraph sync in June 2026** — so StoryGraph is where the e-reader crowd is + heading, which raises the value of us having it and lowers the novelty of Goodreads. + + Verdict: both buildable via cookie-replay with no password storage. Gate behind an explicit + experimental opt-in; expect breakage on site changes and cookie expiry. Prefer official APIs if + they ever appear (StoryGraph has discussed one). + +### Tier 3 — Cookie-replay possible, but higher blast radius / harder. Spike, don't commit. + +- **Amazon Kindle (`read.amazon.com`)** — cookie-replay is proven (`Xetera/kindle-api` reads + library + reading-progress % using `at-main`/`sess-at-main`/`x-main`/`ubid-main`/`session-id` + cookies, valid ~1 year). Two real obstacles beyond Tier 2: (a) **Amazon added TLS fingerprinting + in July 2023**, so a naive server fetch is blocked — you need a browser-mimicking TLS client + (bogdanfinn/tls-client style) or you route through the user's browser via an extension; (b) an + Amazon session cookie is higher blast radius than a Goodreads one (same account as payments, + though scoped to the `read.amazon.com` subdomain in Readwise's model). Whispersync itself (the + device progress protocol) remains private with no endpoint; what's reachable is the Cloud Reader + progress % and the `/notebook` highlights. So "bidirectional Kindle progress" is partially real + (read progress %, write via the same web surface) but engineering-heavy and ToS-gray. +- **Audible** — only a community reverse-engineered API; audiobook position ≠ ebook position + (needs a timestamp↔percentage model). Lower priority. + + Verdict: technically reachable via the same cookie-replay pattern (no password storage), but the + Amazon TLS-fingerprinting workaround and the larger credential blast radius make this a research + spike gated on a security review — not a committed v1 feature. **For highlights specifically, + prefer the Readwise hop (Tier 1) over building this at all.** A direct Kindle connector is only + justified by reading-*progress* sync, which Readwise can't provide. The framework accommodates + it; we don't rush it. + +## What this means for the build + +1. **Build the hub + connector framework** (generalized tables, registry, queue worker, matcher + runner, web-UI pairing screen). Durable regardless of which connectors follow. +2. **Ship Hardcover + Readwise** as the first Tier-1 connectors (token APIs, no extension needed). +3. **Build the browser extension** — the decided, shared credential-capture path for every + cookie-based connector. It's the prerequisite for all of Tier 2/3, so it comes before them. +4. **Gate Tier 2** (Goodreads/StoryGraph, via the extension) behind an explicit experimental opt-in + if there's demand; expect maintenance cost and breakage. Revisit if official APIs appear. +5. **Do not build Tier 3 credential storage.** Track the Amazon/Audible landscape; if an official + API or a legal-reviewed narrow importer becomes viable, the framework already accommodates it. + +## Cookie-replay mechanics (Tier 2/3) + +Shared shape for every cookie-replay connector: + +1. **Credential capture — via a first-party browser extension (decided).** The extension is the + committed capture path for all cookie-based connectors: the user logs into the service in their + own browser, and the extension harvests *only* the specific cookie names below and POSTs them to + us. This is strictly better than paste-the-cookie (no DevTools spelunking, no accidental + over-sharing, matches Readwise's proven UX) and never touches the login page or password. + Manual cookie paste stays as a no-extension fallback only. We store the harvested bundle + encrypted, treated as password-equivalent. Cookie sets per service: + - Goodreads: `_session_id2` (+ `ccsid`) + - StoryGraph: `_story_graph_session` + `remember_user_token` + - Kindle: `at-main`, `sess-at-main`, `x-main`, `ubid-main`, `session-id` +2. **CSRF handshake (Rails sites: Goodreads, StoryGraph).** Before any write, GET an authenticated + HTML page, scrape `` (or the hidden `authenticity_token` input), and + send it as `X-CSRF-Token` (AJAX) or an `authenticity_token` form field (form POST), alongside + the session cookie. +3. **Write** to the service's web endpoint (below). +4. **Expiry handling.** On a redirect-to-login / 401 / signup-page-HTML response, mark the + connector `needs_reauth` and surface it in the status endpoint — never retry-loop a dead + session. Kindle cookies last ~1 year; Goodreads/StoryGraph session cookies are shorter. + +### Goodreads write-path reference + +We can't use the Goodreads API (dead keys), but the **field names** from the still-maintained +Calibre "Goodreads Sync" plugin (`kiwidude68/calibre_plugins → goodreads_sync/core.py`) are the +best-documented reference, because Goodreads' web forms and its old API share parameter naming. +Capture the *current* web/AJAX paths from DevTools while shelving/rating/updating on goodreads.com, +then map onto these known field shapes: + +| Operation | Params (urlencoded) | +|---|---| +| Add/remove shelf | `name=`, `book_id=` (+ `a=remove`) | +| Rating + read date | `review[rating]`, `review[read_at]=YYYY-MM-DD`, `review[review]` | +| Reading progress | `user_status[book_id]`, `user_status[percent]` (or `[page]`), `user_status[body]` | + +Book matching is title/author search (server-side, as everywhere in this doc): Goodreads +`/search/search.xml`-style query, or ISBN if ever available. **Caveat:** the `user_status` progress +update is historically the flakiest Goodreads endpoint (intermittent 401s, "success but no visible +change") — treat progress as best-effort and rely on shelf + read-date as the durable signal. + +StoryGraph writes follow the same GET-CSRF-then-POST shape; `storygraph.koplugin` is the reference +for the current status/progress endpoints. + +## Bidirectional sync (fan-in) notes + +Only connectors with a read capability can push external changes back. The merge rule extends the +existing kosync "newest wins across devices" to "newest wins across sources" — every canonical +event carries a source and timestamp; the hub applies the most recent and re-fans-out to the +others, with loop suppression (don't echo a change back to the source that produced it). This is +straightforward for Tier 1 and mostly irrelevant for Tier 2/3, which are write-mostly or +unavailable. + +## Security posture (applies to all connectors) + +- **We never store passwords.** Tier 2/3 use cookie-replay: the server holds session cookies, not + credentials, and never sees the plaintext password (nor 2FA, since cookies are post-login). +- All external credentials (tokens and cookie bundles) encrypted at rest (`AES-256-GCM`, key from + `TOKEN_ENC_KEY`); connector disabled if the key is unset (self-host default). Never logged, + redacted in errors. +- **Cookie bundles are unscopeable full-session secrets** — treat them as sensitive as passwords + even though they aren't passwords: minimal retention, encrypt, and lean on the user's natural + revocation levers (password change / "log out everywhere" invalidates them). A leaked Amazon + cookie is higher blast radius than a Goodreads one; that's the extra weight on Tier 3. +- **The browser extension is the decided capture model** for cookie connectors (harvest only the + named cookies, like Readwise) — strictly better privacy than paste-the-cookie and than ever + touching the login page. Paste-cookie is the no-extension fallback. The extension is a shared + dependency of every Tier 2/3 connector, so it's the first thing to build before any of them ship. +- Outbound requests restricted to each connector's known hosts (no user-supplied URLs; no SSRF). + Kindle additionally needs a browser-fingerprint-matching TLS client to get past Amazon's 2023 + TLS fingerprinting. + +## Open questions + +- Web UI is a hard prerequisite for pairing anything (OAuth redirects, token paste) — this is the + forcing function that makes the web UI real. Sequence accordingly. +- Per-connector completion/rating mapping differs (star scales, half-stars, DNF states) — define a + canonical rating model once, adapt per connector. +- Do we expose the master account as its own login (email/password/OAuth) distinct from the kosync + credential, so a user can manage pairings without device credentials? Likely yes, once the web + UI exists. diff --git a/migrations/0003_connectors.sql b/migrations/0003_connectors.sql new file mode 100644 index 0000000..563f94a --- /dev/null +++ b/migrations/0003_connectors.sql @@ -0,0 +1,47 @@ +-- Master-sync-hub connector framework. One row set per external service +-- (Hardcover, Readwise, ...), keyed by connector_id. See docs/design/sync-hub.md. + +CREATE TABLE connector_accounts ( + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, -- 'hardcover' | 'readwise' | ... + cred_enc TEXT NOT NULL, -- AES-256-GCM encrypted credential JSON + account_label TEXT, -- display name/email from the service, if known + status TEXT NOT NULL DEFAULT 'ok', -- 'ok' | 'needs_reauth' | 'error' + enabled INTEGER NOT NULL DEFAULT 1, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, connector_id) +); + +CREATE TABLE connector_matches ( + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, + document TEXT NOT NULL, + external_id TEXT, -- service book id (NULL = unmatched) + external_edition TEXT, -- optional edition id + confidence REAL NOT NULL DEFAULT 0, + source TEXT NOT NULL DEFAULT 'none', -- 'auto' | 'manual' | 'none' + query_used TEXT, -- what we searched, for the review UI + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, connector_id, document) +); +CREATE INDEX idx_conn_matches_doc ON connector_matches(user_id, document); + +CREATE TABLE connector_queue ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + connector_id TEXT NOT NULL, + document TEXT NOT NULL, + kind TEXT NOT NULL, -- 'progress' | 'finished' | 'highlight' + payload TEXT NOT NULL, -- JSON event body + attempts INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', -- 'pending' | 'done' | 'dead' + next_try_at INTEGER NOT NULL, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + -- Coalesce: a newer event of the same kind for the same book replaces the pending one. + UNIQUE (user_id, connector_id, document, kind) +); +CREATE INDEX idx_conn_queue_ready ON connector_queue(status, next_try_at); diff --git a/migrations/0004_master_accounts.sql b/migrations/0004_master_accounts.sql new file mode 100644 index 0000000..aba2c3a --- /dev/null +++ b/migrations/0004_master_accounts.sql @@ -0,0 +1,16 @@ +-- Master ("general login") accounts. The website identity you sign into, kept +-- separate from the kosync sync credential your reader uses. A kosync account +-- (users row) is linked to a master account as its native sync identity; the +-- reading data and connectors stay keyed to the kosync user. See docs/design. + +CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + handle TEXT NOT NULL UNIQUE, + token_hash TEXT NOT NULL, -- PBKDF2(MD5(master token)) + created_at INTEGER NOT NULL +); + +-- Link a kosync user to its owning master account. NULL = orphan kosync account +-- (e.g. created directly on a device), linkable later. One kosync per master. +ALTER TABLE users ADD COLUMN account_id INTEGER REFERENCES accounts(id); +CREATE UNIQUE INDEX idx_users_account ON users(account_id) WHERE account_id IS NOT NULL; diff --git a/src/app.ts b/src/app.ts index 31f4274..b33e71e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,34 +1,53 @@ import { Hono } from 'hono'; import type { DB } from './db/db.js'; import type { Config } from './config.js'; -import { authMiddleware, type AppEnv } from './auth/middleware.js'; +import { sessionOrKeyAuth, type AppEnv } from './auth/middleware.js'; import { kosyncRoutes } from './routes/kosync.js'; +import { authRoutes } from './routes/auth.js'; +import { accountRoutes } from './routes/account.js'; +import { webRoutes } from './routes/web.js'; import { progressRoutes } from './routes/v1/progress.js'; import { bookmarkRoutes } from './routes/v1/bookmarks.js'; import { clippingRoutes } from './routes/v1/clippings.js'; import { statsRoutes } from './routes/v1/stats.js'; import { documentRoutes } from './routes/v1/documents.js'; +import { connectorRoutes } from './routes/v1/connectors.js'; +import type { HttpTransport } from './connectors/types.js'; // Injected at build time via package.json; read lazily to keep this file dependency-free. export const VERSION = process.env.npm_package_version ?? '0.1.0'; -export function createApp(db: DB, config: Config): Hono { +export interface AppOptions { + /** Override the connector HTTP transport (tests inject a fake). */ + connectorTransport?: HttpTransport; +} + +export function createApp(db: DB, config: Config, opts: AppOptions = {}): Hono { const app = new Hono(); app.get('/healthz', (c) => c.json({ status: 'ok', version: VERSION })); - // kosync-compatible API at the root — stock KOReader and current CrossPoint + // Web UI (landing / account pages). + app.route('/', webRoutes()); + + // kosync-compatible API at the root - stock KOReader and current CrossPoint // firmware work by changing only the server URL. app.route('/', kosyncRoutes(db, config)); - // Extended CrossPoint API; same auth headers. + // Web account session auth (browser signup/login) + kosync link management. + app.route('/auth', authRoutes(db, config)); + app.route('/account', accountRoutes(db, config)); + + // Extended CrossPoint API; accepts either the web session cookie or the + // device x-auth headers. const v1 = new Hono(); - v1.use('*', authMiddleware(db)); + v1.use('*', sessionOrKeyAuth(db)); v1.route('/', progressRoutes(db)); v1.route('/', bookmarkRoutes(db)); v1.route('/', clippingRoutes(db)); v1.route('/', statsRoutes(db)); v1.route('/', documentRoutes(db)); + v1.route('/', connectorRoutes(db, opts.connectorTransport)); app.route('/api/v1', v1); return app; diff --git a/src/auth/middleware.ts b/src/auth/middleware.ts index 11e7798..224dbe2 100644 --- a/src/auth/middleware.ts +++ b/src/auth/middleware.ts @@ -1,15 +1,25 @@ import type { Context, MiddlewareHandler } from 'hono'; +import { getCookie } from 'hono/cookie'; import type { DB } from '../db/db.js'; import { verifyKey } from './password.js'; +import { SESSION_COOKIE, verifySession } from './session.js'; export interface AuthedUser { id: number; username: string; } +export interface AuthedAccount { + id: number; + handle: string; +} + export type AppEnv = { Variables: { + /** The kosync sync identity that owns the reading data (device or resolved from web session). */ user: AuthedUser; + /** The master ("general login") account, set on web-session routes. */ + account: AuthedAccount; }; }; @@ -54,6 +64,57 @@ export function authMiddleware(db: DB): MiddlewareHandler { }; } +/** + * Require a valid master ("general login") session cookie. Sets `account`. + * Used for the /account management surface (kosync link, master token rotate). + */ +export function masterAuth(db: DB): MiddlewareHandler { + const getAccount = db.prepare('SELECT id, handle FROM accounts WHERE id = ?'); + return async (c, next) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return kosyncError(c, 401, 2001, 'Not signed in'); + const row = getAccount.get(session.uid) as { id: number; handle: string } | undefined; + if (!row) return kosyncError(c, 401, 2001, 'Not signed in'); + c.set('account', { id: row.id, handle: row.handle }); + await next(); + }; +} + +/** + * Accept EITHER a web session cookie OR the device x-auth headers, resolving to + * the kosync sync identity that owns the reading data. Used for the /api/v1 + * surface so both the browser and the firmware reach the same endpoints: + * - device: x-auth headers -> the kosync user directly. + * - web: master session cookie -> the account's linked native kosync user. + * A signed-in master account with no kosync account linked yet gets 409 (the + * web UI prompts to create/link one before showing data). + */ +export function sessionOrKeyAuth(db: DB): MiddlewareHandler { + const headerAuth = authMiddleware(db); + const getAccount = db.prepare('SELECT id, handle FROM accounts WHERE id = ?'); + const getKosyncForAccount = db.prepare( + 'SELECT id, username FROM users WHERE account_id = ?' + ); + return async (c, next) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (session) { + const account = getAccount.get(session.uid) as { id: number; handle: string } | undefined; + if (account) { + c.set('account', account); + const kosync = getKosyncForAccount.get(account.id) as + | { id: number; username: string } + | undefined; + if (!kosync) { + return c.json({ code: 2005, message: 'No sync account linked' }, 409); + } + c.set('user', kosync); + return next(); + } + } + return headerAuth(c, next); + }; +} + /** Minimal in-memory per-IP fixed-window rate limiter. */ export function rateLimiter(limitPerMinute: number): MiddlewareHandler { const hits = new Map(); diff --git a/src/auth/session.ts b/src/auth/session.ts new file mode 100644 index 0000000..4ad4635 --- /dev/null +++ b/src/auth/session.ts @@ -0,0 +1,85 @@ +import crypto from 'node:crypto'; + +/** + * Stateless signed session cookies for the web account. Format: + * base64url(JSON{uid, exp}) + '.' + base64url(HMAC-SHA256(payload)) + * No session table - the HMAC makes the cookie unforgeable. Signed with + * SESSION_SECRET (falls back to TOKEN_ENC_KEY, then an ephemeral per-process key + * so zero-config still works, at the cost of sessions dropping on restart). + */ + +const COOKIE_NAME = 'cp_session'; +const DEFAULT_TTL_SECONDS = 60 * 60 * 24 * 30; // 30 days + +let cachedSecret: Buffer | undefined; +let warnedEphemeral = false; + +function sessionSecret(env: NodeJS.ProcessEnv = process.env): Buffer { + if (cachedSecret) return cachedSecret; + const raw = env.SESSION_SECRET || env.TOKEN_ENC_KEY; + if (raw) { + cachedSecret = crypto.createHash('sha256').update(raw).digest(); + } else { + cachedSecret = crypto.randomBytes(32); + if (!warnedEphemeral) { + warnedEphemeral = true; + console.warn( + JSON.stringify({ + msg: 'no SESSION_SECRET/TOKEN_ENC_KEY set; using an ephemeral key; sessions drop on restart', + }) + ); + } + } + return cachedSecret; +} + +export function resetSessionSecretCache(): void { + cachedSecret = undefined; +} + +export const SESSION_COOKIE = COOKIE_NAME; + +function b64url(buf: Buffer): string { + return buf.toString('base64url'); +} + +export function signSession( + userId: number, + ttlSeconds = DEFAULT_TTL_SECONDS, + env: NodeJS.ProcessEnv = process.env +): string { + const exp = Math.floor(Date.now() / 1000) + ttlSeconds; + const payload = b64url(Buffer.from(JSON.stringify({ uid: userId, exp }))); + const sig = b64url(crypto.createHmac('sha256', sessionSecret(env)).update(payload).digest()); + return `${payload}.${sig}`; +} + +export function verifySession( + token: string | undefined, + env: NodeJS.ProcessEnv = process.env +): { uid: number } | null { + if (!token) return null; + const dot = token.indexOf('.'); + if (dot <= 0) return null; + const payload = token.slice(0, dot); + const sig = token.slice(dot + 1); + const expected = b64url( + crypto.createHmac('sha256', sessionSecret(env)).update(payload).digest() + ); + const a = Buffer.from(sig); + const b = Buffer.from(expected); + if (a.length !== b.length || !crypto.timingSafeEqual(a, b)) return null; + try { + const data = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { + uid: number; + exp: number; + }; + if (typeof data.uid !== 'number' || typeof data.exp !== 'number') return null; + if (data.exp < Math.floor(Date.now() / 1000)) return null; + return { uid: data.uid }; + } catch { + return null; + } +} + +export const SESSION_TTL_SECONDS = DEFAULT_TTL_SECONDS; diff --git a/src/connectors/bookfusion.ts b/src/connectors/bookfusion.ts new file mode 100644 index 0000000..724ac30 --- /dev/null +++ b/src/connectors/bookfusion.ts @@ -0,0 +1,201 @@ +import { decideMatch, extractTitleAuthor, type Candidate } from './matching.js'; +import type { + Connector, + Credential, + DeviceLinkPoll, + DeviceLinkStart, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * BookFusion connector (Tier 3, experimental). Uses BookFusion's OAuth 2.0 + * device-authorization grant to obtain a per-user access token, then pushes + * reading position to /api/user/books/{id}/reading_position. Endpoints and the + * api_version header come from BookFusion's official KOReader plugin. + * + * !!! LIVE-VERIFY GATE !!! + * BookFusion's API is real but not formally public (a developer portal is "coming"). + * The plugin hardcodes client_id "koreader"; confirm we may reuse it or register + * our own before shipping. Reconfirm the device-code, search, and reading_position + * shapes against a live account. Search this file for GATE. + */ + +const BASE = 'https://www.bookfusion.com'; +const API_VERSION = 'application/json; api_version=10'; +// GATE: confirm we can use this client_id; BookFusion may require registration. +const CLIENT_ID = 'koreader'; + +interface BookFusionCred extends Credential { + access_token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as BookFusionCred).access_token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing bookfusion token'); + return t; +} + +function authHeaders(token: string): Record { + return { authorization: `Bearer ${token}`, accept: API_VERSION, 'content-type': 'application/json' }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + // GATE: confirm a cheap authenticated endpoint for validation. + const res = await http(`${BASE}/api/user/books/search`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ query: '', per_page: 1 }), + }); + if (res.status === 401 || res.status === 403) return { ok: false, error: 'invalid token' }; + if (res.status >= 200 && res.status < 300) return { ok: true }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +async function match( + cred: Credential, + doc: DocumentMeta, + http: HttpTransport +): Promise { + const ta = extractTitleAuthor(doc); + if (!ta) return null; + const token = tokenOf(cred); + const q = `${ta.title} ${ta.author}`.trim(); + // GATE: confirm the search request/response shape. + const res = await http(`${BASE}/api/user/books/search`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ query: q, per_page: 10 }), + }); + if (res.status < 200 || res.status >= 300) return null; + let body: any; + try { + body = await res.json(); + } catch { + return null; + } + const hits = extractBooks(body); + if (hits.length === 0) return null; + const decision = decideMatch(ta.title, ta.author, hits); + if (!decision.accepted || !decision.best) return null; + return { externalId: decision.best.externalId, confidence: decision.best.score, queryUsed: q }; +} + +/** GATE: adapt to the real search payload. */ +export function extractBooks(body: any): Candidate[] { + const arr = Array.isArray(body) ? body : (body?.books ?? body?.results ?? []); + const out: Candidate[] = []; + for (const b of Array.isArray(arr) ? arr : []) { + const id = b?.id ?? b?.book_id; + const title = b?.title; + if (id == null || typeof title !== 'string') continue; + const author = b?.author ?? (Array.isArray(b?.authors) ? b.authors[0]?.name ?? b.authors[0] : undefined); + out.push({ externalId: String(id), title, author }); + } + return out; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const token = tokenOf(cred); + const percentage = Math.max(0, Math.min(1, ev.percentage ?? 0)) * 100; // BookFusion uses 0..100 + // GATE: confirm reading_position field names (percentage, page_position_in_book, cfi). + const res = await http(`${BASE}/api/user/books/${m.externalId}/reading_position`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ percentage: Number(percentage.toFixed(4)) }), + }); + if (res.status === 401 || res.status === 403) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + if (res.status >= 200 && res.status < 300) return { ok: true }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; +} + +// --- Device-code linking flow ------------------------------------------------- + +async function beginLink(http: HttpTransport): Promise { + const res = await http(`${BASE}/api/user/auth/device`, { + method: 'POST', + headers: { accept: API_VERSION, 'content-type': 'application/json' }, + body: JSON.stringify({ client_id: CLIENT_ID }), + }); + const body = (await res.json()) as { + device_code: string; + user_code: string; + verification_uri: string; + interval?: number; + expires_in?: number; + }; + return { + deviceCode: body.device_code, + userCode: body.user_code, + verificationUri: body.verification_uri, + interval: body.interval ?? 5, + expiresIn: body.expires_in ?? 900, + }; +} + +async function pollLink(deviceCode: string, http: HttpTransport): Promise { + const res = await http(`${BASE}/api/user/auth/token`, { + method: 'POST', + headers: { accept: API_VERSION, 'content-type': 'application/json' }, + body: JSON.stringify({ + grant_type: 'urn:ietf:params:oauth:grant-type:device_code', + client_id: CLIENT_ID, + device_code: deviceCode, + }), + }); + let body: any = {}; + try { + body = await res.json(); + } catch { + /* ignore */ + } + if (body?.access_token) { + return { status: 'ok', credential: { access_token: body.access_token } }; + } + switch (body?.error) { + case 'authorization_pending': + case 'slow_down': + return { status: 'pending' }; + case 'access_denied': + return { status: 'denied', error: 'You declined the request on BookFusion.' }; + case 'expired_token': + return { status: 'expired', error: 'The code expired. Start again.' }; + default: + return res.status >= 500 + ? { status: 'pending' } + : { status: 'error', error: body?.error ?? `status ${res.status}` }; + } +} + +export const bookfusionConnector: Connector = { + id: 'bookfusion', + displayName: 'BookFusion', + tier: 3, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'device_code', + experimental: true, + validate, + match, + push, + beginLink, + pollLink, +}; diff --git a/src/connectors/fanout.ts b/src/connectors/fanout.ts new file mode 100644 index 0000000..a690e43 --- /dev/null +++ b/src/connectors/fanout.ts @@ -0,0 +1,66 @@ +import type { DB } from '../db/db.js'; +import { secretsEnabled } from '../crypto/secrets.js'; +import { getConnector } from './registry.js'; +import { enqueue } from './queue.js'; +import { activeConnectorIds } from './store.js'; +import type { OutboundEvent } from './types.js'; + +/** + * Enqueue canonical reading events to every linked connector that carries the + * event's kind. Best-effort and synchronous-but-cheap (DB inserts only); the + * queue worker does the network I/O. Never throws into the request path. + */ +function fanOut(db: DB, userId: number, ev: OutboundEvent, coalesceKey?: string): void { + if (!secretsEnabled()) return; + try { + for (const connectorId of activeConnectorIds(db, userId)) { + const conn = getConnector(connectorId); + if (!conn || !conn.capabilities.write || !conn.carries.includes(ev.kind)) continue; + enqueue(db, userId, connectorId, ev, coalesceKey); + } + } catch (err) { + console.error( + JSON.stringify({ msg: 'fanout enqueue failed', error: err instanceof Error ? err.message : String(err) }) + ); + } +} + +export function fanOutProgress( + db: DB, + userId: number, + document: string, + percentage: number, + timestamp: number, + progress?: string, + positionJson?: string | null +): void { + const finished = percentage >= 0.98; + let position: Record | null = null; + if (positionJson) { + try { + position = JSON.parse(positionJson) as Record; + } catch { + position = null; + } + } + fanOut(db, userId, { + kind: finished ? 'finished' : 'progress', + document, + percentage, + progress, + position, + timestamp, + }); +} + +export function fanOutHighlight( + db: DB, + userId: number, + document: string, + clippingId: string, + h: NonNullable, + timestamp: number +): void { + // Per-clipping coalesce key so distinct highlights on one book each queue. + fanOut(db, userId, { kind: 'highlight', document, timestamp, highlight: h }, `highlight:${clippingId}`); +} diff --git a/src/connectors/hardcover.ts b/src/connectors/hardcover.ts new file mode 100644 index 0000000..3f88c89 --- /dev/null +++ b/src/connectors/hardcover.ts @@ -0,0 +1,192 @@ +import { decideMatch, extractTitleAuthor, type Candidate } from './matching.js'; +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * Hardcover connector (Tier 1). Public GraphQL API, per-user bearer token. + * Carries reading progress + shelf status. + * + * !!! LIVE-VERIFY GATE !!! + * Hardcover's API is beta. The GraphQL operations below (field names, the + * `me`/search shapes, and the user_book mutation names/status ids) are modeled + * from the documented schema but MUST be checked against the live GraphQL + * explorer at https://hardcover.app/account/api before enabling in production. + * Every network call is funneled through gql() so the exact queries live in one + * place and are covered by fixture tests. Search the file for GATE to find each + * spot that needs confirmation. + */ + +const ENDPOINT = 'https://api.hardcover.app/v1/graphql'; + +// GATE: confirm Hardcover's user_book status ids (want-to-read/reading/read). +const STATUS_READING = 2; +const STATUS_READ = 3; + +interface HardcoverCred extends Credential { + token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as HardcoverCred).token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing hardcover token'); + return t; +} + +async function gql( + http: HttpTransport, + token: string, + query: string, + variables: Record +): Promise<{ data?: any; errors?: { message: string }[]; status: number }> { + const res = await http(ENDPOINT, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: token.startsWith('Bearer ') ? token : `Bearer ${token}`, + }, + body: JSON.stringify({ query, variables }), + }); + if (res.status === 401 || res.status === 403) { + return { status: res.status, errors: [{ message: 'unauthorized' }] }; + } + let body: any = {}; + try { + body = await res.json(); + } catch { + body = {}; + } + return { status: res.status, data: body.data, errors: body.errors }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + // GATE: confirm the `me` query shape. + const r = await gql(http, token, `query { me { username } }`, {}); + if (r.status === 401 || r.status === 403) return { ok: false, error: 'invalid token' }; + if (r.errors?.length) return { ok: false, error: r.errors[0].message }; + const username = r.data?.me?.[0]?.username ?? r.data?.me?.username; + return { ok: true, accountLabel: username ?? undefined }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +async function match( + cred: Credential, + doc: DocumentMeta, + http: HttpTransport +): Promise { + const ta = extractTitleAuthor(doc); + if (!ta) return null; + const token = tokenOf(cred); + const q = `${ta.title} ${ta.author}`.trim(); + // GATE: confirm Hardcover's search query name and result shape. + const r = await gql( + http, + token, + `query Search($q: String!) { + search(query: $q, query_type: "Book", per_page: 10) { + results + } + }`, + { q } + ); + if (r.errors?.length || !r.data) return null; + const hits = extractSearchHits(r.data); + if (hits.length === 0) return null; + const decision = decideMatch(ta.title, ta.author, hits); + if (!decision.accepted || !decision.best) return null; + return { + externalId: decision.best.externalId, + confidence: decision.best.score, + queryUsed: q, + }; +} + +/** GATE: adapt to the real search payload. Handles a couple of plausible shapes. */ +export function extractSearchHits(data: any): Candidate[] { + const raw = + data?.search?.results?.hits ?? + data?.search?.results ?? + data?.search ?? + []; + const arr = Array.isArray(raw) ? raw : Array.isArray(raw?.hits) ? raw.hits : []; + const out: Candidate[] = []; + for (const h of arr) { + const doc = h?.document ?? h; + const id = doc?.id ?? doc?.book_id; + const title = doc?.title; + if (id == null || typeof title !== 'string') continue; + const author = + doc?.author_names?.[0] ?? + doc?.contributions?.[0]?.author?.name ?? + doc?.author ?? + undefined; + out.push({ + externalId: String(id), + title, + author, + popularity: typeof doc?.users_count === 'number' ? doc.users_count : undefined, + }); + } + return out; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const token = tokenOf(cred); + const bookId = Number(m.externalId); + const finished = ev.kind === 'finished' || (ev.percentage ?? 0) >= 0.999; + const status = finished ? STATUS_READ : STATUS_READING; + + // GATE: confirm the upsert mutation name/args. Hardcover uses an + // insert_user_book / update_user_book pattern keyed by book + status. + const mutation = ` + mutation SetStatus($bookId: Int!, $status: Int!) { + insert_user_book(object: { book_id: $bookId, status_id: $status }) { + id + } + }`; + const r = await gql(http, token, mutation, { bookId, status }); + + if (r.status === 401 || r.status === 403) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (r.status === 429) { + return { ok: false, retryable: true, error: 'rate limited' }; + } + if (r.errors?.length) { + // GraphQL validation errors won't fix themselves on retry. + return { ok: false, retryable: false, error: r.errors[0].message }; + } + if (r.status >= 500) { + return { ok: false, retryable: true, error: `server ${r.status}` }; + } + return { ok: true }; +} + +export const hardcoverConnector: Connector = { + id: 'hardcover', + displayName: 'Hardcover', + tier: 1, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'token', + experimental: false, + validate, + match, + push, +}; diff --git a/src/connectors/kosync.ts b/src/connectors/kosync.ts new file mode 100644 index 0000000..0524797 --- /dev/null +++ b/src/connectors/kosync.ts @@ -0,0 +1,129 @@ +import crypto from 'node:crypto'; +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * External kosync mirror connector. Forwards reading progress to ANOTHER kosync + * server (e.g. sync.koreader.rocks, a friend's server, or a second CrossPoint + * Sync instance) so other KOReader devices see it too. + * + * This is the simplest connector: kosync keys progress by the same 32-hex + * document hash we already have, so there is NO book matching. It also speaks + * the exact protocol we store, so mirroring is lossless (we can forward the rich + * position + metadata superset; a plain kosync target ignores the extras). + * + * Outbound only for now (we push to them); fan-in would need loop suppression. + */ + +interface KosyncCred extends Credential { + server: string; + username: string; + password: string; +} + +function parseCred(cred: Credential): KosyncCred | null { + const server = typeof cred.server === 'string' ? cred.server.trim() : ''; + const username = typeof cred.username === 'string' ? cred.username.trim() : ''; + const password = typeof cred.password === 'string' ? cred.password : ''; + if (!server || !username || !password) return null; + return { server, username, password }; +} + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +/** Normalize a server URL: add scheme if missing, strip trailing slashes. */ +export function baseUrl(server: string): string { + let url = server.includes('://') ? server : `https://${server}`; + while (url.endsWith('/')) url = url.slice(0, -1); + return url; +} + +function authHeaders(c: KosyncCred): Record { + return { + 'x-auth-user': c.username, + 'x-auth-key': md5(c.password), + accept: 'application/vnd.koreader.v1+json', + 'content-type': 'application/json', + }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + const c = parseCred(cred); + if (!c) return { ok: false, error: 'server, username and password are required' }; + try { + const res = await http(`${baseUrl(c.server)}/users/auth`, { + method: 'GET', + headers: authHeaders(c), + }); + if (res.status === 200) return { ok: true, accountLabel: `${c.username} @ ${c.server}` }; + if (res.status === 401) return { ok: false, error: 'invalid sync credentials' }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +// The document hash is the kosync document id: matching is identity, no network. +async function match(_cred: Credential, doc: DocumentMeta, _http: HttpTransport): Promise { + return { externalId: doc.document, confidence: 1 }; +} + +async function push( + cred: Credential, + m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + const c = parseCred(cred); + if (!c) return { ok: false, retryable: false, needsReauth: true, error: 'bad credential' }; + + const percentage = Math.max(0, Math.min(1, ev.percentage ?? 0)); + const body: Record = { + document: m.externalId, + // kosync servers reject an empty progress string; fall back to the percentage. + progress: ev.progress && ev.progress.length > 0 ? ev.progress : String(percentage), + percentage, + device: 'CrossPoint Sync', + device_id: 'crosspoint-sync-mirror', + }; + // Forward the rich position superset (a plain kosync target ignores it). + if (ev.position) body.position = ev.position; + + try { + const res = await http(`${baseUrl(c.server)}/syncs/progress`, { + method: 'PUT', + headers: authHeaders(c), + body: JSON.stringify(body), + }); + if (res.status === 200 || res.status === 202) return { ok: true }; + if (res.status === 401) return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, retryable: true, error: err instanceof Error ? err.message : String(err) }; + } +} + +export const kosyncConnector: Connector = { + id: 'kosync', + displayName: 'Another KOSync server', + tier: 1, + capabilities: { read: false, write: true }, + carries: ['progress', 'finished'], + credentialKind: 'kosync', + experimental: false, + validate, + match, + push, +}; diff --git a/src/connectors/matching.ts b/src/connectors/matching.ts new file mode 100644 index 0000000..718d685 --- /dev/null +++ b/src/connectors/matching.ts @@ -0,0 +1,144 @@ +import type { DocumentMeta } from './types.js'; + +/** + * Shared, connector-agnostic book-matching helpers. Connectors call their own + * search API, then use scoreCandidate() to rank results against the document's + * title/author. Pure functions - unit-tested independently of any network. + */ + +/** Fold diacritics, lowercase, drop punctuation, collapse whitespace. */ +export function normalizeText(s: string): string { + return s + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') // combining marks + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** Strip subtitle after a colon and trailing "(Series, Book 2)" style suffixes. */ +export function coreTitle(title: string): string { + let t = title.replace(/\s*[:—-]\s.*$/, ''); // subtitle + t = t.replace(/\s*\((?:[^)]*\b(?:book|vol|volume|part|no)\b[^)]*)\)\s*$/i, ''); + return t.trim(); +} + +/** "Last, First" -> "First Last"; also normalizes. */ +export function normalizeAuthor(author: string): string { + const trimmed = author.trim(); + const comma = trimmed.indexOf(','); + if (comma > 0 && trimmed.indexOf(',') === trimmed.lastIndexOf(',')) { + const last = trimmed.slice(0, comma).trim(); + const first = trimmed.slice(comma + 1).trim(); + return normalizeText(`${first} ${last}`); + } + return normalizeText(trimmed); +} + +/** + * Derive {title, author} for book matching. The EPUB's own title/author (which + * the firmware extracts and sends in the progress `metadata` object) is the real + * signal - this is the primary and expected path. + * + * Filename is only a last resort for the rare title-less case (a malformed EPUB + * whose getTitle() was empty). Note it can't rescue the "no metadata at all" + * case: filename ships in the same metadata object as title/author, so if we + * lack title we usually lack filename too. We deliberately do NOT guess + * "Title - Author" vs "Author - Title" ordering - we drop the separators and let + * the whole string be a fuzzy search query, which search engines handle fine. + */ +export function extractTitleAuthor(doc: DocumentMeta): { title: string; author: string } | null { + if (doc.title) { + return { title: doc.title, author: doc.author ?? '' }; + } + if (doc.filename) { + const base = doc.filename.replace(/\.[a-z0-9]+$/i, '').replace(/\s*-\s*/g, ' ').trim(); + return base ? { title: base, author: '' } : null; + } + return null; +} + +/** Token set overlap (Jaccard) of two normalized strings. */ +function tokenOverlap(a: string, b: string): number { + const sa = new Set(a.split(' ').filter(Boolean)); + const sb = new Set(b.split(' ').filter(Boolean)); + if (sa.size === 0 || sb.size === 0) return 0; + let inter = 0; + for (const t of sa) if (sb.has(t)) inter++; + return inter / (sa.size + sb.size - inter); +} + +export interface Candidate { + externalId: string; + title: string; + author?: string; + /** Optional popularity/rank signal (higher = more popular), used as a tiebreak. */ + popularity?: number; +} + +export interface ScoredCandidate extends Candidate { + score: number; +} + +/** + * Score a candidate against the wanted title/author. Title similarity dominates; + * author overlap is a strong secondary that disambiguates same-title collisions. + */ +export function scoreCandidate( + wantTitle: string, + wantAuthor: string, + cand: Candidate +): number { + const wt = normalizeText(coreTitle(wantTitle)); + const ct = normalizeText(coreTitle(cand.title)); + const titleScore = wt && ct ? tokenOverlap(wt, ct) : 0; + + const wa = wantAuthor ? normalizeAuthor(wantAuthor) : ''; + const ca = cand.author ? normalizeAuthor(cand.author) : ''; + const authorScore = wa && ca ? tokenOverlap(wa, ca) : 0; + + // No author info on either side: rely on title alone (capped so it can't + // clear a high threshold on title-only, since title collisions are common). + if (!wa || !ca) return titleScore * 0.85; + return titleScore * 0.7 + authorScore * 0.3; +} + +export interface MatchDecision { + best: ScoredCandidate | null; + accepted: boolean; +} + +/** + * Rank candidates and decide whether to auto-accept: the top must clear + * `threshold` AND beat the runner-up by `margin` (unless the runner-up is the + * same book - same normalized title+author - in which case ambiguity between + * editions is fine and we take the more popular one). + */ +export function decideMatch( + wantTitle: string, + wantAuthor: string, + candidates: Candidate[], + opts: { threshold?: number; margin?: number } = {} +): MatchDecision { + const threshold = opts.threshold ?? 0.6; + const margin = opts.margin ?? 0.15; + if (candidates.length === 0) return { best: null, accepted: false }; + + const scored: ScoredCandidate[] = candidates + .map((c) => ({ ...c, score: scoreCandidate(wantTitle, wantAuthor, c) })) + .sort((a, b) => b.score - a.score || (b.popularity ?? 0) - (a.popularity ?? 0)); + + const best = scored[0]; + if (best.score < threshold) return { best, accepted: false }; + + const runner = scored[1]; + if (!runner) return { best, accepted: true }; + + const sameBook = + normalizeText(coreTitle(best.title)) === normalizeText(coreTitle(runner.title)) && + normalizeAuthor(best.author ?? '') === normalizeAuthor(runner.author ?? ''); + if (sameBook) return { best, accepted: true }; + + return { best, accepted: best.score - runner.score >= margin }; +} diff --git a/src/connectors/queue.ts b/src/connectors/queue.ts new file mode 100644 index 0000000..dd14898 --- /dev/null +++ b/src/connectors/queue.ts @@ -0,0 +1,115 @@ +import { withTransaction, type DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import type { OutboundEvent } from './types.js'; + +/** + * Coalescing retry queue for connector fan-out. One pending row per + * (user, connector, document, kind): a newer event of the same kind replaces + * the older pending one, so bursts of progress syncs collapse to the latest. + */ + +const MAX_ATTEMPTS = 8; +// Exponential backoff in seconds, capped; index by attempt count. +const BACKOFF = [30, 120, 300, 900, 3600, 10800, 43200, 86400]; + +export interface QueueRow { + id: number; + user_id: number; + connector_id: string; + document: string; + kind: string; + payload: string; + attempts: number; + status: string; + next_try_at: number; + last_error: string | null; +} + +/** + * Enqueue an event. The queue `kind` column doubles as the coalescing key: a + * later event with the same (user, connector, document, kind) replaces the + * pending one. Progress uses the bare event kind (collapse bursts to latest); + * highlights pass a per-item coalesceKey so distinct highlights on the same book + * don't overwrite each other. The stored payload always carries the real + * OutboundEvent, whose `.kind` drives connector dispatch. + */ +export function enqueue( + db: DB, + userId: number, + connectorId: string, + ev: OutboundEvent, + coalesceKey: string = ev.kind, + now = nowSeconds() +): void { + db.prepare( + `INSERT INTO connector_queue + (user_id, connector_id, document, kind, payload, attempts, status, next_try_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 0, 'pending', ?, ?, ?) + ON CONFLICT(user_id, connector_id, document, kind) DO UPDATE SET + payload = excluded.payload, + attempts = 0, + status = 'pending', + next_try_at = excluded.next_try_at, + last_error = NULL, + updated_at = excluded.updated_at` + ).run(userId, connectorId, ev.document, coalesceKey, JSON.stringify(ev), now, now, now); +} + +export function claimReady(db: DB, limit: number, now = nowSeconds()): QueueRow[] { + return db + .prepare( + `SELECT id, user_id, connector_id, document, kind, payload, attempts, status, next_try_at, last_error + FROM connector_queue + WHERE status = 'pending' AND next_try_at <= ? + ORDER BY next_try_at + LIMIT ?` + ) + .all(now, limit) as unknown as QueueRow[]; +} + +export function markDone(db: DB, id: number, now = nowSeconds()): void { + db.prepare(`UPDATE connector_queue SET status = 'done', updated_at = ? WHERE id = ?`).run(now, id); +} + +/** Record a failed attempt: reschedule with backoff, or dead-letter past the cap. */ +export function markFailed( + db: DB, + row: QueueRow, + error: string, + retryable: boolean, + now = nowSeconds() +): void { + const attempts = row.attempts + 1; + if (!retryable || attempts >= MAX_ATTEMPTS) { + db.prepare( + `UPDATE connector_queue SET status = 'dead', attempts = ?, last_error = ?, updated_at = ? WHERE id = ?` + ).run(attempts, error.slice(0, 500), now, row.id); + return; + } + const backoff = BACKOFF[Math.min(attempts - 1, BACKOFF.length - 1)]; + db.prepare( + `UPDATE connector_queue SET attempts = ?, last_error = ?, next_try_at = ?, updated_at = ? WHERE id = ?` + ).run(attempts, error.slice(0, 500), now + backoff, now, row.id); +} + +/** Drop a queued row entirely (e.g. connector unlinked). */ +export function purgeConnector(db: DB, userId: number, connectorId: string): void { + db.prepare(`DELETE FROM connector_queue WHERE user_id = ? AND connector_id = ?`).run( + userId, + connectorId + ); +} + +export function queueDepth(db: DB, userId: number, connectorId: string): { pending: number; dead: number } { + const row = db + .prepare( + `SELECT + SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending, + SUM(CASE WHEN status = 'dead' THEN 1 ELSE 0 END) AS dead + FROM connector_queue WHERE user_id = ? AND connector_id = ?` + ) + .get(userId, connectorId) as { pending: number | null; dead: number | null }; + return { pending: row.pending ?? 0, dead: row.dead ?? 0 }; +} + +export const _internals = { MAX_ATTEMPTS, BACKOFF, withTransaction }; diff --git a/src/connectors/readwise.ts b/src/connectors/readwise.ts new file mode 100644 index 0000000..e345084 --- /dev/null +++ b/src/connectors/readwise.ts @@ -0,0 +1,157 @@ +import type { + Connector, + Credential, + DocumentMeta, + HttpTransport, + Match, + OutboundEvent, + PushResult, + ValidateResult, +} from './types.js'; + +/** + * Readwise connector (Tier 1). Official REST API, per-user access token. + * Carries highlights/notes only - NOT reading progress. Bidirectional: + * - fan-out: push CrossInk clippings via POST /api/v2/highlights/ + * - fan-in: pull highlights via GET /api/v2/export/ (incl. Kindle, which + * Readwise ingests for us - the "aggregator hop", see docs/design/sync-hub.md). + * + * LIVE-VERIFY GATE: the v2 field names below follow Readwise's documented API + * (readwise.io/api_deets) but should be reconfirmed at implementation. Endpoints + * are stable and public, so the risk is lower than Hardcover's beta API. + */ + +const BASE = 'https://readwise.io/api/v2'; + +interface ReadwiseCred extends Credential { + token: string; +} + +function tokenOf(cred: Credential): string { + const t = (cred as ReadwiseCred).token; + if (typeof t !== 'string' || t.length === 0) throw new Error('missing readwise token'); + return t; +} + +function authHeaders(token: string): Record { + return { authorization: `Token ${token}`, 'content-type': 'application/json' }; +} + +async function validate(cred: Credential, http: HttpTransport): Promise { + try { + const token = tokenOf(cred); + const res = await http(`${BASE}/auth/`, { method: 'GET', headers: authHeaders(token) }); + if (res.status === 204 || res.status === 200) return { ok: true }; + if (res.status === 401) return { ok: false, error: 'invalid token' }; + return { ok: false, error: `unexpected status ${res.status}` }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +/** + * Readwise groups highlights by (title, author) rather than an external book id + * we look up ahead of time - when we push, we send title/author and Readwise + * creates/finds the book. So "matching" is trivial: any document with a title + * (or a parseable filename) is pushable. We store a synthetic match so the + * runner treats it as matched. + */ +async function match( + cred: Credential, + doc: DocumentMeta, + _http: HttpTransport +): Promise { + const title = doc.title ?? titleFromFilename(doc.filename); + if (!title) return null; + return { externalId: `title:${title}`, confidence: 1, queryUsed: title }; +} + +function titleFromFilename(filename: string | null): string | null { + if (!filename) return null; + const base = filename.replace(/\.[a-z0-9]+$/i, ''); + const parts = base.split(' - '); + return (parts[0] ?? base).trim() || null; +} + +async function push( + cred: Credential, + _m: Match, + ev: OutboundEvent, + http: HttpTransport +): Promise { + // Readwise only accepts highlight events; progress/finished are no-ops here. + if (ev.kind !== 'highlight' || !ev.highlight) { + return { ok: true }; + } + const token = tokenOf(cred); + const h = ev.highlight; + const highlight: Record = { + text: h.text, + title: h.title ?? undefined, + author: h.author ?? undefined, + source_type: 'crosspoint', + category: 'books', + note: h.note ?? undefined, + location: h.location ?? undefined, + location_type: 'order', + highlighted_at: h.highlightedAt ? new Date(h.highlightedAt * 1000).toISOString() : undefined, + }; + const res = await http(`${BASE}/highlights/`, { + method: 'POST', + headers: authHeaders(token), + body: JSON.stringify({ highlights: [highlight] }), + }); + if (res.status === 401) { + return { ok: false, retryable: false, needsReauth: true, error: 'unauthorized' }; + } + if (res.status === 429) return { ok: false, retryable: true, error: 'rate limited' }; + if (res.status >= 500) return { ok: false, retryable: true, error: `server ${res.status}` }; + if (res.status === 200 || res.status === 201) return { ok: true }; + return { ok: false, retryable: false, error: `unexpected status ${res.status}` }; +} + +/** + * Fan-in: export highlights updated since a cursor. Returns raw Readwise books + * (each with a highlights[] array) plus the nextCursor for incremental pulls. + * The caller maps these into the canonical clippings store. Kept separate from + * the Connector interface (which is fan-out only for now) - wired when clipping + * fan-in lands. + */ +export async function exportHighlights( + cred: Credential, + http: HttpTransport, + updatedAfter?: number, + pageCursor?: string +): Promise<{ results: unknown[]; nextCursor: string | null; status: number }> { + const token = tokenOf(cred); + const params = new URLSearchParams(); + if (updatedAfter) params.set('updatedAfter', new Date(updatedAfter * 1000).toISOString()); + if (pageCursor) params.set('pageCursor', pageCursor); + const res = await http(`${BASE}/export/?${params.toString()}`, { + method: 'GET', + headers: authHeaders(token), + }); + if (res.status !== 200) return { results: [], nextCursor: null, status: res.status }; + const body = (await res.json()) as { results?: unknown[]; nextPageCursor?: string | null }; + return { + results: body.results ?? [], + nextCursor: body.nextPageCursor ?? null, + status: res.status, + }; +} + +export const readwiseConnector: Connector = { + id: 'readwise', + displayName: 'Readwise', + tier: 1, + capabilities: { read: true, write: true }, + carries: ['highlight'], + credentialKind: 'token', + experimental: false, + // Hidden until the fan-in hop is wired and firmware sends clippings; the code + // stays registered so re-enabling is a one-line change. + hidden: true, + validate, + match, + push, +}; diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts new file mode 100644 index 0000000..85d0333 --- /dev/null +++ b/src/connectors/registry.ts @@ -0,0 +1,34 @@ +import type { Connector, HttpTransport } from './types.js'; +import { hardcoverConnector } from './hardcover.js'; +import { readwiseConnector } from './readwise.js'; +import { kosyncConnector } from './kosync.js'; +import { bookfusionConnector } from './bookfusion.js'; + +/** All connectors known to this build. */ +const CONNECTORS: Connector[] = [ + kosyncConnector, + hardcoverConnector, + readwiseConnector, + bookfusionConnector, +]; + +const byId = new Map(CONNECTORS.map((c) => [c.id, c])); + +/** Connectors shown in the UI/API (excludes hidden ones). */ +export function listConnectors(): Connector[] { + return CONNECTORS.filter((c) => !c.hidden); +} + +export function getConnector(id: string): Connector | undefined { + return byId.get(id); +} + +/** Default transport: the platform fetch, adapted to HttpTransport. */ +export const fetchTransport: HttpTransport = async (url, init) => { + const res = await fetch(url, init); + return { + status: res.status, + text: () => res.text(), + json: () => res.json(), + }; +}; diff --git a/src/connectors/runner.ts b/src/connectors/runner.ts new file mode 100644 index 0000000..d578679 --- /dev/null +++ b/src/connectors/runner.ts @@ -0,0 +1,139 @@ +import type { DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import { getConnector, fetchTransport } from './registry.js'; +import { + claimReady, + markDone, + markFailed, + type QueueRow, +} from './queue.js'; +import { + decryptCredential, + documentMeta, + getAccount, + getMatch, + saveMatch, + setAccountStatus, +} from './store.js'; +import type { HttpTransport, Match, OutboundEvent } from './types.js'; + +/** + * Resolve a connector match for a document, using the cached row when present. + * Manual matches are authoritative and never recomputed. Missing/auto rows are + * (re)computed via the connector's own search. Returns null when unmatched. + */ +export async function resolveMatch( + db: DB, + connectorId: string, + userId: number, + document: string, + http: HttpTransport +): Promise { + const cached = getMatch(db, userId, connectorId, document); + if (cached && cached.source === 'manual') { + return cached.external_id + ? { + externalId: cached.external_id, + externalEdition: cached.external_edition, + confidence: cached.confidence, + } + : null; + } + if (cached && cached.external_id) { + return { + externalId: cached.external_id, + externalEdition: cached.external_edition, + confidence: cached.confidence, + }; + } + + const connector = getConnector(connectorId); + const account = getAccount(db, userId, connectorId); + if (!connector || !account) return null; + const cred = decryptCredential(account); + const meta = documentMeta(db, userId, document); + const match = await connector.match(cred, meta, http); + saveMatch(db, userId, connectorId, document, match, match ? 'auto' : 'none'); + return match; +} + +/** Process a single queued event. Returns true if handled (done or dead). */ +export async function processRow(db: DB, row: QueueRow, http: HttpTransport): Promise { + const connector = getConnector(row.connector_id); + const account = getAccount(db, row.user_id, row.connector_id); + if (!connector || !account || !account.enabled) { + // Connector gone or disabled - drop permanently. + markFailed(db, row, 'connector unavailable or disabled', false); + return; + } + if (account.status === 'needs_reauth') { + markFailed(db, row, 'account needs reauth', false); + return; + } + + let match: Match | null; + try { + match = await resolveMatch(db, row.connector_id, row.user_id, row.document, http); + } catch (err) { + markFailed(db, row, `match failed: ${errStr(err)}`, true); + return; + } + if (!match) { + // Unmatched documents can't be pushed; drop this event (a later manual + // match + fresh sync will re-enqueue). Not an error state. + markFailed(db, row, 'no book match', false); + return; + } + + const ev = JSON.parse(row.payload) as OutboundEvent; + const cred = decryptCredential(account); + try { + const result = await connector.push(cred, match, ev, http); + if (result.ok) { + markDone(db, row.id); + return; + } + if (result.needsReauth) { + setAccountStatus(db, row.user_id, row.connector_id, 'needs_reauth', result.error); + } + markFailed(db, row, result.error, result.retryable); + } catch (err) { + markFailed(db, row, errStr(err), true); + } +} + +/** Drain up to `limit` ready events. Returns the number processed. */ +export async function drainQueue( + db: DB, + http: HttpTransport = fetchTransport, + limit = 20, + now = nowSeconds() +): Promise { + const rows = claimReady(db, limit, now); + for (const row of rows) { + await processRow(db, row, http); + } + return rows.length; +} + +function errStr(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Start a periodic drain loop; returns a stop function. */ +export function startQueueWorker(db: DB, intervalMs = 15_000): () => void { + let running = false; + const timer = setInterval(async () => { + if (running) return; // never overlap drains + running = true; + try { + await drainQueue(db); + } catch (err) { + console.error(JSON.stringify({ msg: 'queue drain error', error: errStr(err) })); + } finally { + running = false; + } + }, intervalMs); + if (typeof timer.unref === 'function') timer.unref(); + return () => clearInterval(timer); +} diff --git a/src/connectors/store.ts b/src/connectors/store.ts new file mode 100644 index 0000000..1855d94 --- /dev/null +++ b/src/connectors/store.ts @@ -0,0 +1,170 @@ +import type { DB } from '../db/db.js'; +import { nowSeconds } from '../models/sync.js'; +import { decryptSecret, encryptSecret } from '../crypto/secrets.js'; +import type { Credential, DocumentMeta, Match } from './types.js'; + +export interface AccountRow { + user_id: number; + connector_id: string; + cred_enc: string; + account_label: string | null; + status: string; + enabled: number; + last_error: string | null; + created_at: number; + updated_at: number; +} + +export function upsertAccount( + db: DB, + userId: number, + connectorId: string, + cred: Credential, + accountLabel: string | null, + now = nowSeconds() +): void { + const enc = encryptSecret(JSON.stringify(cred)); + db.prepare( + `INSERT INTO connector_accounts + (user_id, connector_id, cred_enc, account_label, status, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, 'ok', 1, ?, ?) + ON CONFLICT(user_id, connector_id) DO UPDATE SET + cred_enc = excluded.cred_enc, + account_label = excluded.account_label, + status = 'ok', + enabled = 1, + last_error = NULL, + updated_at = excluded.updated_at` + ).run(userId, connectorId, enc, accountLabel, now, now); +} + +export function getAccount(db: DB, userId: number, connectorId: string): AccountRow | null { + return ( + (db + .prepare('SELECT * FROM connector_accounts WHERE user_id = ? AND connector_id = ?') + .get(userId, connectorId) as AccountRow | undefined) ?? null + ); +} + +export function listAccounts(db: DB, userId: number): AccountRow[] { + return db + .prepare('SELECT * FROM connector_accounts WHERE user_id = ?') + .all(userId) as unknown as AccountRow[]; +} + +export function decryptCredential(row: AccountRow): Credential { + return JSON.parse(decryptSecret(row.cred_enc)) as Credential; +} + +export function deleteAccount(db: DB, userId: number, connectorId: string): void { + db.prepare('DELETE FROM connector_accounts WHERE user_id = ? AND connector_id = ?').run( + userId, + connectorId + ); +} + +export function setAccountStatus( + db: DB, + userId: number, + connectorId: string, + status: string, + error: string | null, + now = nowSeconds() +): void { + db.prepare( + 'UPDATE connector_accounts SET status = ?, last_error = ?, updated_at = ? WHERE user_id = ? AND connector_id = ?' + ).run(status, error, now, userId, connectorId); +} + +/** All connector accounts that are linked, enabled, and healthy for fan-out. */ +export function activeConnectorIds(db: DB, userId: number): string[] { + return ( + db + .prepare( + `SELECT connector_id FROM connector_accounts WHERE user_id = ? AND enabled = 1 AND status != 'error'` + ) + .all(userId) as { connector_id: string }[] + ).map((r) => r.connector_id); +} + +export interface MatchRow { + user_id: number; + connector_id: string; + document: string; + external_id: string | null; + external_edition: string | null; + confidence: number; + source: string; + query_used: string | null; + updated_at: number; +} + +export function getMatch( + db: DB, + userId: number, + connectorId: string, + document: string +): MatchRow | null { + return ( + (db + .prepare( + 'SELECT * FROM connector_matches WHERE user_id = ? AND connector_id = ? AND document = ?' + ) + .get(userId, connectorId, document) as MatchRow | undefined) ?? null + ); +} + +export function saveMatch( + db: DB, + userId: number, + connectorId: string, + document: string, + match: Match | null, + source: 'auto' | 'manual' | 'none', + now = nowSeconds() +): void { + db.prepare( + `INSERT INTO connector_matches + (user_id, connector_id, document, external_id, external_edition, confidence, source, query_used, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(user_id, connector_id, document) DO UPDATE SET + external_id = excluded.external_id, + external_edition = excluded.external_edition, + confidence = excluded.confidence, + source = excluded.source, + query_used = excluded.query_used, + updated_at = excluded.updated_at` + ).run( + userId, + connectorId, + document, + match?.externalId ?? null, + match?.externalEdition ?? null, + match?.confidence ?? 0, + source, + match?.queryUsed ?? null, + now + ); +} + +export function listMatches(db: DB, userId: number, connectorId: string): MatchRow[] { + return db + .prepare( + 'SELECT * FROM connector_matches WHERE user_id = ? AND connector_id = ? ORDER BY updated_at DESC' + ) + .all(userId, connectorId) as unknown as MatchRow[]; +} + +export function documentMeta(db: DB, userId: number, document: string): DocumentMeta { + const row = db + .prepare('SELECT title, author, filename FROM documents WHERE user_id = ? AND document = ?') + .get(userId, document) as + | { title: string | null; author: string | null; filename: string | null } + | undefined; + return { + document, + title: row?.title ?? null, + author: row?.author ?? null, + filename: row?.filename ?? null, + }; +} diff --git a/src/connectors/types.ts b/src/connectors/types.ts new file mode 100644 index 0000000..8f04689 --- /dev/null +++ b/src/connectors/types.ts @@ -0,0 +1,125 @@ +/** + * Connector framework types. A connector adapts crosspoint-sync's canonical + * reading state to one external service. See docs/design/sync-hub.md. + */ + +export type Capability = { read: boolean; write: boolean }; + +/** What data types a connector carries (progress/shelves vs highlights). */ +export type DataKind = 'progress' | 'finished' | 'highlight'; + +export type CredentialKind = 'token' | 'oauth' | 'cookies' | 'kosync' | 'device_code'; + +/** Interactive OAuth device-code link handshake (BookFusion). */ +export interface DeviceLinkStart { + deviceCode: string; + userCode: string; + verificationUri: string; + interval: number; + expiresIn: number; +} +export interface DeviceLinkPoll { + status: 'pending' | 'ok' | 'denied' | 'expired' | 'error'; + credential?: Record; + accountLabel?: string; + error?: string; +} + +/** Feasibility/trust tier from the design doc. */ +export type Tier = 1 | 2 | 3; + +/** Metadata we know about a document, used for server-side book matching. */ +export interface DocumentMeta { + document: string; + title: string | null; + author: string | null; + filename: string | null; +} + +export interface Match { + externalId: string; + externalEdition?: string | null; + confidence: number; // 0..1 + queryUsed?: string; +} + +/** A canonical reading event to fan out to a connector. */ +export interface OutboundEvent { + kind: DataKind; + document: string; + /** 0..1 reading fraction (progress/finished events). */ + percentage?: number; + /** kosync progress string (xpath/CFI). Present on progress events; used by the kosync mirror. */ + progress?: string; + /** Rich CrossPoint position, forwarded losslessly by the kosync mirror. */ + position?: Record | null; + /** unix seconds when this happened on the device/server. */ + timestamp: number; + /** For highlight events. */ + highlight?: { + text: string; + note?: string | null; + title?: string | null; + author?: string | null; + location?: number | null; + highlightedAt?: number | null; + }; +} + +/** Result of a push attempt; retryable=false means don't re-queue (permanent). */ +export type PushResult = + | { ok: true } + | { ok: false; retryable: boolean; error: string; needsReauth?: boolean }; + +export interface ValidateResult { + ok: boolean; + accountLabel?: string; + error?: string; +} + +/** + * Minimal HTTP transport injected into connectors so tests can supply a fake + * without real network access. Mirrors the subset of fetch we use. + */ +export interface HttpTransport { + (url: string, init: { + method: string; + headers?: Record; + body?: string; + }): Promise<{ + status: number; + text(): Promise; + json(): Promise; + }>; +} + +/** A parsed credential (shape is connector-specific; stored encrypted as JSON). */ +export type Credential = Record; + +export interface Connector { + id: string; + displayName: string; + tier: Tier; + capabilities: Capability; + /** Which data kinds this connector accepts on fan-out. */ + carries: DataKind[]; + credentialKind: CredentialKind; + /** Whether the connector is experimental (Tier 2/3 cookie-replay). */ + experimental: boolean; + /** Hidden from the connector list/UI (still registered; not user-linkable via the UI). */ + hidden?: boolean; + + /** Validate a credential and return the account label if possible. */ + validate(cred: Credential, http: HttpTransport): Promise; + + /** Resolve a document to an external book id. Null = no confident match. */ + match(cred: Credential, doc: DocumentMeta, http: HttpTransport): Promise; + + /** Push one outbound event. Only called for write-capable connectors. */ + push(cred: Credential, match: Match, ev: OutboundEvent, http: HttpTransport): Promise; + + /** Begin an interactive device-code link (OAuth device grant). Optional. */ + beginLink?(http: HttpTransport): Promise; + /** Poll a device-code link until it completes. Optional. */ + pollLink?(deviceCode: string, http: HttpTransport): Promise; +} diff --git a/src/crypto/secrets.ts b/src/crypto/secrets.ts new file mode 100644 index 0000000..bbbe02d --- /dev/null +++ b/src/crypto/secrets.ts @@ -0,0 +1,76 @@ +import crypto from 'node:crypto'; + +/** + * Symmetric encryption for connector credentials (third-party tokens and session + * cookie bundles) at rest. AES-256-GCM with a random per-record IV; the auth tag + * detects tampering. The key comes from TOKEN_ENC_KEY - if it's unset, all + * connectors are disabled rather than storing secrets in the clear. + * + * Wire format (base64 of): [1-byte version][12-byte iv][16-byte tag][ciphertext] + */ + +const VERSION = 1; +const IV_LEN = 12; +const TAG_LEN = 16; + +let cachedKey: Buffer | null | undefined; + +/** Resolves the 32-byte key from TOKEN_ENC_KEY (hex, base64, or raw >=32 chars), or null. */ +export function getEncryptionKey(env: NodeJS.ProcessEnv = process.env): Buffer | null { + if (cachedKey !== undefined) return cachedKey; + const raw = env.TOKEN_ENC_KEY; + if (!raw) { + cachedKey = null; + return null; + } + let key: Buffer | null = null; + if (/^[0-9a-fA-F]{64}$/.test(raw)) { + key = Buffer.from(raw, 'hex'); + } else { + const b64 = Buffer.from(raw, 'base64'); + if (b64.length === 32) { + key = b64; + } else if (Buffer.byteLength(raw) >= 32) { + // Derive a stable 32-byte key from an arbitrary passphrase. + key = crypto.createHash('sha256').update(raw).digest(); + } + } + if (!key || key.length !== 32) { + throw new Error('TOKEN_ENC_KEY must be 32 bytes (64 hex chars, base64, or a >=32 char passphrase)'); + } + cachedKey = key; + return cachedKey; +} + +/** Test seam: forget the cached key so a changed env is picked up. */ +export function resetEncryptionKeyCache(): void { + cachedKey = undefined; +} + +export function secretsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return getEncryptionKey(env) !== null; +} + +export function encryptSecret(plaintext: string, env: NodeJS.ProcessEnv = process.env): string { + const key = getEncryptionKey(env); + if (!key) throw new Error('TOKEN_ENC_KEY not configured; connector credentials cannot be stored'); + const iv = crypto.randomBytes(IV_LEN); + const cipher = crypto.createCipheriv('aes-256-gcm', key, iv); + const enc = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); + const tag = cipher.getAuthTag(); + return Buffer.concat([Buffer.from([VERSION]), iv, tag, enc]).toString('base64'); +} + +export function decryptSecret(stored: string, env: NodeJS.ProcessEnv = process.env): string { + const key = getEncryptionKey(env); + if (!key) throw new Error('TOKEN_ENC_KEY not configured; cannot decrypt connector credentials'); + const buf = Buffer.from(stored, 'base64'); + if (buf.length < 1 + IV_LEN + TAG_LEN) throw new Error('Malformed encrypted secret'); + if (buf[0] !== VERSION) throw new Error(`Unsupported secret version ${buf[0]}`); + const iv = buf.subarray(1, 1 + IV_LEN); + const tag = buf.subarray(1 + IV_LEN, 1 + IV_LEN + TAG_LEN); + const enc = buf.subarray(1 + IV_LEN + TAG_LEN); + const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(enc), decipher.final()]).toString('utf8'); +} diff --git a/src/index.ts b/src/index.ts index 00b5719..c874680 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,8 @@ import { serve } from '@hono/node-server'; import { createApp } from './app.js'; import { fromEnv } from './config.js'; import { migrate, openDatabase } from './db/db.js'; +import { secretsEnabled } from './crypto/secrets.js'; +import { startQueueWorker } from './connectors/runner.js'; const DATABASE_PATH = process.env.DATABASE_PATH ?? '/data/crosspoint.db'; const PORT = Number(process.env.PORT ?? 8080); @@ -11,12 +13,20 @@ migrate(db); const app = createApp(db, fromEnv()); +// Connector fan-out queue worker (only meaningful when encryption - hence +// connectors - is configured). +const connectorsEnabled = secretsEnabled(); +if (connectorsEnabled) { + startQueueWorker(db); +} + serve({ fetch: app.fetch, port: PORT }, (info) => { console.log( JSON.stringify({ msg: 'crosspoint-sync listening', port: info.port, db: DATABASE_PATH, + connectors: connectorsEnabled ? 'enabled' : 'disabled (no TOKEN_ENC_KEY)', }) ); }); diff --git a/src/models/position.ts b/src/models/position.ts index 74c8950..f662980 100644 --- a/src/models/position.ts +++ b/src/models/position.ts @@ -1,5 +1,5 @@ /** - * Rich reading position — 1:1 with the firmware's CompactPosition wire struct + * Rich reading position - 1:1 with the firmware's CompactPosition wire struct * (see CrossInk NearbyBookPositionSyncActivity.h). Page fields are layout hints; * pctQ / para / anchor / xpath are the portable parts. */ diff --git a/src/models/stats.ts b/src/models/stats.ts index 7975751..21a23b0 100644 --- a/src/models/stats.ts +++ b/src/models/stats.ts @@ -1,7 +1,7 @@ /** * Reading-stats snapshots mirror CrossInk's GlobalReadingStats / BookReadingStats * (stats_v5). Each device uploads its own snapshot; snapshots are never merged into - * each other — the server aggregates across devices on read, same model as the + * each other - the server aggregates across devices on read, same model as the * firmware's nearby P2P stats sync. */ diff --git a/src/routes/account.ts b/src/routes/account.ts new file mode 100644 index 0000000..181780a --- /dev/null +++ b/src/routes/account.ts @@ -0,0 +1,180 @@ +import crypto from 'node:crypto'; +import { Hono } from 'hono'; +import { deleteCookie } from 'hono/cookie'; +import { withTransaction, type DB } from '../db/db.js'; +import type { Config } from '../config.js'; +import { masterAuth, type AppEnv } from '../auth/middleware.js'; +import { hashKey, verifyKey } from '../auth/password.js'; +import { invalidateAuthCache } from '../auth/middleware.js'; +import { SESSION_COOKIE } from '../auth/session.js'; +import { nowSeconds } from '../models/sync.js'; +import { USERNAME_RE } from './kosync.js'; + +/** Permanently delete a kosync user and every row of its reading data. */ +function deleteKosyncUserData(db: DB, userId: number, username: string): void { + withTransaction(db, () => { + for (const table of [ + 'connector_queue', + 'connector_matches', + 'connector_accounts', + 'stats_device_book', + 'stats_device_global', + 'clippings', + 'bookmarks', + 'documents', + 'progress', + ]) { + db.prepare(`DELETE FROM ${table} WHERE user_id = ?`).run(userId); + } + db.prepare('DELETE FROM users WHERE id = ?').run(userId); + }); + invalidateAuthCache(username); +} + +/** + * Manage the CrossPoint Sync (KOSync) account linked under a master account. + * This is a standard KOReader-compatible sync account: the user picks a username + * and password, exactly like any kosync server, and enters them in their + * reader's KOReader Sync settings. It is where reading data + connectors live. + * A master account owns at most one; it can create a fresh one or link an + * existing (e.g. device-created) one by proving ownership with its password. + * + * The reader sends MD5(password); we store PBKDF2(MD5(password)) in + * users.key_hash, same as any kosync password. The plaintext is never stored. + */ + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +export function accountRoutes(db: DB, config: Config): Hono { + const app = new Hono(); + app.use('*', masterAuth(db)); + + // Status of this account's linked kosync sync identity. + app.get('/kosync', (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT username FROM users WHERE account_id = ?') + .get(account.id) as { username: string } | undefined; + return c.json({ linked: !!row, username: row?.username ?? null }); + }); + + // Create a fresh CrossPoint Sync (KOSync) account with a user-chosen username + // and password, then link it. Standard kosync: the reader uses the same creds. + app.post('/kosync', async (c) => { + const account = c.get('account'); + if (db.prepare('SELECT 1 FROM users WHERE account_id = ?').get(account.id)) { + return c.json({ error: 'A sync account is already linked' }, 409); + } + let username: string | null = null; + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + username = typeof body.username === 'string' ? body.username.trim() : null; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!username || !USERNAME_RE.test(username)) { + return c.json({ error: 'Invalid username' }, 400); + } + if (!password || password.length < 1 || password.length > 256) { + return c.json({ error: 'Invalid password' }, 400); + } + if (db.prepare('SELECT 1 FROM users WHERE username = ?').get(username)) { + return c.json({ error: 'That sync username is taken' }, 409); + } + db.prepare( + 'INSERT INTO users (username, key_hash, account_id, created_at) VALUES (?, ?, ?, ?)' + ).run(username, hashKey(md5(password)), account.id, nowSeconds()); + return c.json({ username }); + }); + + // Link an existing kosync account by proving ownership with its password. + app.put('/kosync', async (c) => { + const account = c.get('account'); + if (db.prepare('SELECT 1 FROM users WHERE account_id = ?').get(account.id)) { + return c.json({ error: 'A sync account is already linked' }, 409); + } + let username: string | null = null; + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + username = typeof body.username === 'string' ? body.username.trim() : null; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!username || !password) { + return c.json({ error: 'Username and password required' }, 400); + } + const row = db + .prepare('SELECT id, key_hash, account_id FROM users WHERE username = ?') + .get(username) as { id: number; key_hash: string; account_id: number | null } | undefined; + // The device sends MD5(password); the web form takes the plain password. + if (!row || !verifyKey(md5(password), row.key_hash)) { + return c.json({ error: 'Invalid sync account credentials' }, 401); + } + if (row.account_id && row.account_id !== account.id) { + return c.json({ error: 'That sync account is linked to another login' }, 409); + } + db.prepare('UPDATE users SET account_id = ? WHERE id = ?').run(account.id, row.id); + return c.json({ linked: true, username }); + }); + + // Detach the kosync account (data stays; it just becomes an orphan again). + app.delete('/kosync', (c) => { + const account = c.get('account'); + db.prepare('UPDATE users SET account_id = NULL WHERE account_id = ?').run(account.id); + return c.json({ linked: false }); + }); + + // Permanently delete the linked kosync account and ALL its reading data. + app.delete('/kosync/data', (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (!row) return c.json({ error: 'No sync account linked' }, 409); + deleteKosyncUserData(db, row.id, row.username); + return c.json({ deleted: true }); + }); + + // Permanently delete the master account, its linked kosync account, and all + // reading data. Signs the browser out. + app.delete('/', (c) => { + const account = c.get('account'); + const kosync = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (kosync) deleteKosyncUserData(db, kosync.id, kosync.username); + db.prepare('DELETE FROM accounts WHERE id = ?').run(account.id); + deleteCookie(c, SESSION_COOKIE, { path: '/' }); + return c.json({ deleted: true }); + }); + + // Change the password of the linked kosync account (reader must be updated). + app.post('/kosync/password', async (c) => { + const account = c.get('account'); + const row = db + .prepare('SELECT id, username FROM users WHERE account_id = ?') + .get(account.id) as { id: number; username: string } | undefined; + if (!row) return c.json({ error: 'No sync account linked' }, 409); + let password: string | null = null; + try { + const body = (await c.req.json()) as Record; + password = typeof body.password === 'string' ? body.password : null; + } catch { + /* validation below */ + } + if (!password || password.length < 1 || password.length > 256) { + return c.json({ error: 'Invalid password' }, 400); + } + db.prepare('UPDATE users SET key_hash = ? WHERE id = ?').run(hashKey(md5(password)), row.id); + invalidateAuthCache(row.username); + return c.json({ username: row.username }); + }); + + return app; +} diff --git a/src/routes/auth.ts b/src/routes/auth.ts new file mode 100644 index 0000000..3b7cefd --- /dev/null +++ b/src/routes/auth.ts @@ -0,0 +1,136 @@ +import crypto from 'node:crypto'; +import { Hono, type Context } from 'hono'; +import { getCookie, setCookie, deleteCookie } from 'hono/cookie'; +import type { DB } from '../db/db.js'; +import type { Config } from '../config.js'; +import type { AppEnv } from '../auth/middleware.js'; +import { hashKey, verifyKey } from '../auth/password.js'; +import { rateLimiter } from '../auth/middleware.js'; +import { + SESSION_COOKIE, + SESSION_TTL_SECONDS, + signSession, + verifySession, +} from '../auth/session.js'; +import { nowSeconds } from '../models/sync.js'; +import { USERNAME_RE } from './kosync.js'; + +/** + * Master ("general login") accounts. This is the website identity, kept separate + * from the kosync device credential (see account.ts for linking a kosync sync + * account under a master account). There are no passwords: signup issues a + * master token (shown once) that IS the web credential. We store + * PBKDF2(MD5(token)); the session cookie carries the master account id. + * + * Token format: xp1__. The embedded id lets login work from the + * token alone. This token is NOT a kosync secret and never touches a device. + */ + +const TOKEN_RE = /^xp1_(\d+)_[0-9a-f]{32}$/; + +function md5(s: string): string { + return crypto.createHash('md5').update(s).digest('hex'); +} + +function masterToken(accountId: number): string { + return `xp1_${accountId}_${crypto.randomBytes(16).toString('hex')}`; +} + +function setSessionCookie(c: Context, accountId: number) { + const proto = c.req.header('x-forwarded-proto'); + const secure = proto ? proto.split(',')[0].trim() === 'https' : false; + setCookie(c, SESSION_COOKIE, signSession(accountId), { + httpOnly: true, + sameSite: 'Lax', + secure, + path: '/', + maxAge: SESSION_TTL_SECONDS, + }); +} + +export function authRoutes(db: DB, config: Config): Hono { + const app = new Hono(); + + // Create a master account: pick a handle, receive a login token (shown once). + app.post('/signup', rateLimiter(config.authRateLimitPerMinute), async (c) => { + if (config.registrationDisabled) { + return c.json({ error: 'Registration is disabled' }, 403); + } + let handle: string | null = null; + try { + const body = (await c.req.json()) as Record; + handle = typeof body.handle === 'string' ? body.handle.trim() : null; + } catch { + /* validation below */ + } + if (!handle || !USERNAME_RE.test(handle)) { + return c.json({ error: 'Invalid handle' }, 400); + } + if (db.prepare('SELECT 1 FROM accounts WHERE handle = ?').get(handle)) { + return c.json({ error: 'Handle is already taken' }, 409); + } + const secret = crypto.randomBytes(16).toString('hex'); + const info = db + .prepare('INSERT INTO accounts (handle, token_hash, created_at) VALUES (?, ?, ?)') + .run(handle, '', nowSeconds()); + const accountId = Number(info.lastInsertRowid); + const token = `xp1_${accountId}_${secret}`; + db.prepare('UPDATE accounts SET token_hash = ? WHERE id = ?').run(hashKey(md5(token)), accountId); + setSessionCookie(c, accountId); + return c.json({ handle, token }); + }); + + // Web login: paste the master token, get a session cookie. + app.post('/login', rateLimiter(config.authRateLimitPerMinute), async (c) => { + let token: string | null = null; + try { + const body = (await c.req.json()) as Record; + token = typeof body.token === 'string' ? body.token.trim() : null; + } catch { + /* validation below */ + } + const parsed = token?.match(TOKEN_RE); + if (!token || !parsed) return c.json({ error: 'Invalid token' }, 401); + const accountId = Number(parsed[1]); + const row = db + .prepare('SELECT token_hash FROM accounts WHERE id = ?') + .get(accountId) as { token_hash: string } | undefined; + if (!row || !verifyKey(md5(token), row.token_hash)) { + return c.json({ error: 'Invalid token' }, 401); + } + setSessionCookie(c, accountId); + return c.json({ ok: true }); + }); + + app.post('/logout', (c) => { + deleteCookie(c, SESSION_COOKIE, { path: '/' }); + return c.json({ ok: true }); + }); + + app.get('/me', (c) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return c.json({ error: 'Not signed in' }, 401); + const row = db + .prepare('SELECT handle FROM accounts WHERE id = ?') + .get(session.uid) as { handle: string } | undefined; + if (!row) return c.json({ error: 'Not signed in' }, 401); + return c.json({ handle: row.handle }); + }); + + // Rotate the master login token (revokes the old one). + app.post('/token/rotate', (c) => { + const session = verifySession(getCookie(c, SESSION_COOKIE)); + if (!session) return c.json({ error: 'Not signed in' }, 401); + if (!db.prepare('SELECT 1 FROM accounts WHERE id = ?').get(session.uid)) { + return c.json({ error: 'Not signed in' }, 401); + } + const token = masterToken(session.uid); + db.prepare('UPDATE accounts SET token_hash = ? WHERE id = ?').run( + hashKey(md5(token)), + session.uid + ); + return c.json({ token }); + }); + + return app; +} diff --git a/src/routes/kosync.ts b/src/routes/kosync.ts index 8ac4290..061f209 100644 --- a/src/routes/kosync.ts +++ b/src/routes/kosync.ts @@ -11,11 +11,12 @@ import { import { hashKey } from '../auth/password.js'; import { parsePosition } from '../models/position.js'; import { nowSeconds } from '../models/sync.js'; +import { fanOutProgress } from '../connectors/fanout.js'; -const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; +export const USERNAME_RE = /^[A-Za-z0-9._@+-]{1,64}$/; export function isValidDocument(v: unknown): v is string { - // KOReader sends a 32-hex MD5, but the key is opaque — stay lenient. + // KOReader sends a 32-hex MD5, but the key is opaque - stay lenient. return typeof v === 'string' && /^[A-Za-z0-9._-]{1,64}$/.test(v); } @@ -188,6 +189,7 @@ export function kosyncRoutes(db: DB, config: Config): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt, parsed.record.progress, parsed.record.position); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); diff --git a/src/routes/v1/clippings.ts b/src/routes/v1/clippings.ts index efd9f9b..d67e1ad 100644 --- a/src/routes/v1/clippings.ts +++ b/src/routes/v1/clippings.ts @@ -3,6 +3,8 @@ import { withTransaction, type DB } from '../../db/db.js'; import { kosyncError, type AppEnv } from '../../auth/middleware.js'; import { isValidDocument } from '../kosync.js'; import { isItemId, nowSeconds, parseListParams } from '../../models/sync.js'; +import { fanOutHighlight } from '../../connectors/fanout.js'; +import { documentMeta } from '../../connectors/store.js'; const MAX_BATCH = 50; const MAX_TEXT = 2048; // matches the firmware's My Clippings.txt export cap @@ -130,6 +132,8 @@ export function clippingRoutes(db: DB): Hono { type Op = () => void; const ops: Op[] = []; + type Highlight = { id: string; text: string; note: string | null; chapter: string; createdAt: number }; + const highlights: Highlight[] = []; for (const raw of items) { const o = raw as Record; if (!isItemId(o.id)) { @@ -177,10 +181,32 @@ export function clippingRoutes(db: DB): Hono { note, color, createdAt, now ) ); + highlights.push({ id, text, note, chapter, createdAt }); } withTransaction(db, () => { for (const op of ops) op(); }); + // Fan out highlights to connectors that carry them (e.g. Readwise). Best + // effort; the document's title/author (if synced) become the Readwise book. + if (highlights.length > 0) { + const meta = documentMeta(db, user.id, document); + for (const h of highlights) { + fanOutHighlight( + db, + user.id, + document, + h.id, + { + text: h.text, + note: h.note, + title: meta.title, + author: meta.author, + highlightedAt: h.createdAt > 0 ? h.createdAt : null, + }, + now + ); + } + } return c.json({ until: now, accepted: ops.length }); }); diff --git a/src/routes/v1/connectors.ts b/src/routes/v1/connectors.ts new file mode 100644 index 0000000..b4cd6d7 --- /dev/null +++ b/src/routes/v1/connectors.ts @@ -0,0 +1,232 @@ +import { Hono } from 'hono'; +import { withTransaction, type DB } from '../../db/db.js'; +import { kosyncError, type AppEnv } from '../../auth/middleware.js'; +import { secretsEnabled } from '../../crypto/secrets.js'; +import { fetchTransport, getConnector, listConnectors } from '../../connectors/registry.js'; +import { purgeConnector, queueDepth } from '../../connectors/queue.js'; +import { resolveMatch } from '../../connectors/runner.js'; +import { + deleteAccount, + getAccount, + getMatch, + listMatches, + saveMatch, + upsertAccount, +} from '../../connectors/store.js'; +import { isValidDocument } from '../kosync.js'; +import type { HttpTransport } from '../../connectors/types.js'; + +/** + * Master-sync-hub connector management. Same x-auth headers as the rest of v1. + * Credential entry realistically happens from a browser (token paste / OAuth), + * but every endpoint works over curl too. + * + * `transport` is injectable so tests can validate/link connectors without real + * network calls. + */ +export function connectorRoutes(db: DB, transport: HttpTransport = fetchTransport): Hono { + const app = new Hono(); + + // List available connectors + this user's link status. + app.get('/connectors', (c) => { + const user = c.get('user'); + const enabled = secretsEnabled(); + return c.json({ + encryption: enabled ? 'enabled' : 'disabled', + connectors: listConnectors().map((conn) => { + const account = getAccount(db, user.id, conn.id); + return { + id: conn.id, + name: conn.displayName, + tier: conn.tier, + experimental: conn.experimental, + carries: conn.carries, + capabilities: conn.capabilities, + credential_kind: conn.credentialKind, + linked: !!account, + status: account?.status ?? null, + account: account?.account_label ?? null, + queue: account ? queueDepth(db, user.id, conn.id) : undefined, + }; + }), + }); + }); + + // Link (or re-link) a connector by validating and storing its credential. + app.put('/connectors/:id', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!secretsEnabled()) { + return c.json( + { code: 2003, message: 'Server has no TOKEN_ENC_KEY; connector storage disabled' }, + 403 + ); + } + let body: unknown; + try { + body = await c.req.json(); + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const cred = (body as Record | null)?.credential; + if (typeof cred !== 'object' || cred === null) { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const result = await conn.validate(cred as Record, transport); + if (!result.ok) { + return c.json({ code: 2003, message: result.error ?? 'Credential rejected' }, 400); + } + const user = c.get('user'); + upsertAccount(db, user.id, conn.id, cred as Record, result.accountLabel ?? null); + return c.json({ id: conn.id, linked: true, account: result.accountLabel ?? null }); + }); + + // Begin an interactive device-code link (BookFusion). Returns the user code + + // verification URL for the browser to show, and the device code to poll with. + app.post('/connectors/:id/link/begin', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!conn.beginLink) return c.json({ code: 2003, message: 'Connector has no device link' }, 400); + if (!secretsEnabled()) { + return c.json({ code: 2003, message: 'Server has no TOKEN_ENC_KEY; connector storage disabled' }, 403); + } + try { + const start = await conn.beginLink(transport); + return c.json({ + device_code: start.deviceCode, + user_code: start.userCode, + verification_uri: start.verificationUri, + interval: start.interval, + expires_in: start.expiresIn, + }); + } catch (err) { + return c.json({ code: 2003, message: err instanceof Error ? err.message : 'Link failed' }, 502); + } + }); + + // Poll a device-code link; on success, store the credential and link. + app.post('/connectors/:id/link/poll', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + if (!conn.pollLink) return c.json({ code: 2003, message: 'Connector has no device link' }, 400); + let deviceCode: unknown; + try { + deviceCode = ((await c.req.json()) as Record).device_code; + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + if (typeof deviceCode !== 'string') return kosyncError(c, 403, 2003, 'Invalid request'); + try { + const result = await conn.pollLink(deviceCode, transport); + if (result.status === 'ok' && result.credential) { + const user = c.get('user'); + // Confirm the freshly minted credential works, then store it. + const v = await conn.validate(result.credential, transport); + upsertAccount(db, user.id, conn.id, result.credential, v.accountLabel ?? result.accountLabel ?? null); + return c.json({ status: 'ok', linked: true }); + } + return c.json({ status: result.status, error: result.error ?? null }); + } catch (err) { + return c.json({ status: 'error', error: err instanceof Error ? err.message : 'poll failed' }, 502); + } + }); + + // Unlink and wipe queued work + matches. + app.delete('/connectors/:id', (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const user = c.get('user'); + withTransaction(db, () => { + deleteAccount(db, user.id, conn.id); + purgeConnector(db, user.id, conn.id); + db.prepare('DELETE FROM connector_matches WHERE user_id = ? AND connector_id = ?').run( + user.id, + conn.id + ); + }); + return c.json({ id: conn.id, linked: false }); + }); + + // List this user's book matches for a connector (for the review UI). + app.get('/connectors/:id/matches', (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const user = c.get('user'); + return c.json({ + connector: conn.id, + matches: listMatches(db, user.id, conn.id).map((m) => ({ + document: m.document, + external_id: m.external_id, + confidence: m.confidence, + source: m.source, + query_used: m.query_used, + updated_at: m.updated_at, + })), + }); + }); + + // Manually set/override a match (sticky - never auto-recomputed). + app.put('/connectors/:id/matches/:document', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const document = c.req.param('document'); + if (!isValidDocument(document)) { + return kosyncError(c, 403, 2004, "Field 'document' not provided."); + } + let body: unknown; + try { + body = await c.req.json(); + } catch { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + const o = (body ?? {}) as Record; + const externalId = o.external_id; + const user = c.get('user'); + if (externalId === null) { + // Explicit "no match" override - stop trying to sync this document. + saveMatch(db, user.id, conn.id, document, null, 'manual'); + return c.json({ document, external_id: null, source: 'manual' }); + } + if (typeof externalId !== 'string' || externalId.length === 0 || externalId.length > 128) { + return kosyncError(c, 403, 2003, 'Invalid request'); + } + saveMatch( + db, + user.id, + conn.id, + document, + { + externalId, + externalEdition: typeof o.external_edition === 'string' ? o.external_edition : null, + confidence: 1, + }, + 'manual' + ); + return c.json({ document, external_id: externalId, source: 'manual' }); + }); + + // Force (re)matching of a document now - useful for testing and the review UI. + app.post('/connectors/:id/rematch/:document', async (c) => { + const conn = getConnector(c.req.param('id')); + if (!conn) return c.json({ code: 2003, message: 'Unknown connector' }, 404); + const document = c.req.param('document'); + if (!isValidDocument(document)) { + return kosyncError(c, 403, 2004, "Field 'document' not provided."); + } + const user = c.get('user'); + if (!getAccount(db, user.id, conn.id)) { + return c.json({ code: 2003, message: 'Connector not linked' }, 400); + } + // Clear any auto/none row so resolveMatch recomputes (manual is preserved). + const existing = getMatch(db, user.id, conn.id, document); + if (existing && existing.source !== 'manual') { + db.prepare( + 'DELETE FROM connector_matches WHERE user_id = ? AND connector_id = ? AND document = ?' + ).run(user.id, conn.id, document); + } + const match = await resolveMatch(db, conn.id, user.id, document, transport); + return c.json({ document, match: match ?? null }); + }); + + return app; +} diff --git a/src/routes/v1/progress.ts b/src/routes/v1/progress.ts index ee95f34..9a692a7 100644 --- a/src/routes/v1/progress.ts +++ b/src/routes/v1/progress.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import type { DB } from '../../db/db.js'; import { kosyncError, type AppEnv } from '../../auth/middleware.js'; import { isValidDocument, parseProgressBody, upsertProgress } from '../kosync.js'; +import { fanOutProgress } from '../../connectors/fanout.js'; export function progressRoutes(db: DB): Hono { const app = new Hono(); @@ -19,11 +20,12 @@ export function progressRoutes(db: DB): Hono { return kosyncError(c, 403, parsed.code, parsed.message); } upsertProgress(db, parsed.record); + fanOutProgress(db, user.id, parsed.record.document, parsed.record.percentage, parsed.record.updatedAt, parsed.record.progress, parsed.record.position); return c.json({ document: parsed.record.document, timestamp: parsed.record.updatedAt }); }); // List every synced document with its newest progress (joined with any known - // metadata) — lets clients and UIs discover documents without knowing hashes. + // metadata) - lets clients and UIs discover documents without knowing hashes. app.get('/progress', (c) => { const user = c.get('user'); const limitRaw = Number(c.req.query('limit') ?? 100); diff --git a/src/routes/web.ts b/src/routes/web.ts new file mode 100644 index 0000000..5515095 --- /dev/null +++ b/src/routes/web.ts @@ -0,0 +1,492 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Hono } from 'hono'; +import { getCookie } from 'hono/cookie'; +import type { AppEnv } from '../auth/middleware.js'; +import { SESSION_COOKIE, verifySession } from '../auth/session.js'; + +/** + * Minimal server-rendered web UI (no framework, no build step, no deps). Styled + * to match the CrossPoint Reader site: Inter (UI) + Lora (display) + Geist Mono, + * stone neutrals with the forest-green brand accent, white cards on stone-50. + * Pages are static HTML calling the JSON /auth and /api/v1 endpoints via fetch; + * the session cookie authenticates automatically (same-origin). + */ + +const ASSETS_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'assets'); +const LOGO = fs.readFileSync(path.join(ASSETS_DIR, 'logo.png')); +const FAVICON = fs.readFileSync(path.join(ASSETS_DIR, 'favicon.png')); + +const STYLE = ` + :root { + color-scheme: light; + --stone-50:#fafaf9; --stone-100:#f5f5f4; --stone-200:#e7e5e4; --stone-300:#d6d3d1; + --stone-400:#a8a29e; --stone-500:#78716c; --stone-600:#57534e; --stone-700:#44403c; + --stone-900:#1c1917; --stone-950:#0c0a09; + --brand-50:#f0f5f3; --brand-100:#d6e5de; --brand-400:#69917d; --brand-500:#4a7a62; + --brand-600:#3d6652; --brand-700:#315243; + --ring:rgba(12,10,9,0.05); + } + * { box-sizing:border-box; } + html { -webkit-text-size-adjust:100%; } + body { + margin:0; background:var(--stone-50); color:var(--stone-900); + font-family:"InterVariable","Inter",ui-sans-serif,system-ui,sans-serif; + font-feature-settings:"cv02","cv03","cv04","cv11"; + -webkit-font-smoothing:antialiased; + } + .display { font-family:"Lora",ui-serif,serif; } + .mono { font-family:"Geist Mono",ui-monospace,monospace; } + + header.site { + position:sticky; top:0; z-index:40; + border-bottom:1px solid rgba(231,229,228,0.8); + background:rgba(250,250,249,0.8); backdrop-filter:blur(12px); + } + header.site .bar { max-width:64rem; margin:0 auto; padding:14px 20px; + display:flex; align-items:center; justify-content:space-between; gap:16px; } + .wordmark { display:flex; align-items:center; gap:10px; text-decoration:none; } + .wordmark img { width:28px; height:28px; border-radius:6px; display:block; } + .wordmark span { font-family:"Lora",serif; font-weight:600; font-size:16px; + letter-spacing:-0.01em; color:var(--stone-900); } + .wordmark .accent { color:var(--brand-600); } + + .wrap { max-width:32rem; margin:0 auto; padding:44px 20px 96px; } + .center { text-align:center; } + .eyebrow { display:inline-flex; align-items:center; gap:7px; font-size:12px; font-weight:600; + letter-spacing:0.08em; text-transform:uppercase; color:var(--brand-600); } + .eyebrow::before { content:""; width:16px; height:1px; background:var(--brand-400); display:inline-block; } + h1 { font-family:"Lora",serif; font-weight:600; font-size:30px; line-height:1.15; + letter-spacing:-0.015em; margin:14px 0 0; color:var(--stone-900); } + .sub { color:var(--stone-600); margin:12px 0 0; line-height:1.6; } + + .card { background:#fff; border-radius:12px; box-shadow:0 0 0 1px var(--ring); padding:24px; } + .card + .card { margin-top:16px; } + .card h2 { font-size:13px; font-weight:600; text-transform:uppercase; letter-spacing:0.05em; + color:var(--stone-500); margin:0 0 14px; } + + label { display:block; font-size:14px; font-weight:500; color:var(--stone-700); margin:0 0 6px; } + input { width:100%; padding:10px 14px; border:1px solid var(--stone-200); border-radius:8px; + background:var(--stone-50); color:var(--stone-900); font-size:14px; } + input::placeholder { color:var(--stone-400); } + input:focus { outline:none; border-color:var(--brand-400); box-shadow:0 0 0 3px rgba(74,122,98,0.15); } + + button { appearance:none; border:0; border-radius:8px; padding:10px 16px; font-size:14px; + font-weight:600; cursor:pointer; font-family:inherit; } + button.primary { background:var(--brand-500); color:#fff; box-shadow:0 1px 2px rgba(0,0,0,0.06); } + button.primary:hover { background:var(--brand-600); } + button.ghost { background:#fff; color:var(--stone-700); box-shadow:0 0 0 1px var(--stone-200); } + button.ghost:hover { background:var(--stone-50); } + button.danger { background:#fff; color:#b91c1c; box-shadow:0 0 0 1px #f0cccc; } + button.danger:hover { background:#fef2f2; } + button.copied { color:var(--brand-700); box-shadow:0 0 0 1px var(--brand-100); background:var(--brand-50); opacity:1; } + button:disabled { opacity:.5; cursor:default; } + button.copied:disabled { opacity:1; } + button.full { width:100%; } + .mt { margin-top:14px; } + + .row { display:flex; align-items:center; justify-content:space-between; gap:14px; } + .muted { color:var(--stone-500); font-size:13px; line-height:1.55; } + .pill { font-size:11px; font-weight:600; padding:2px 9px; border-radius:999px; + box-shadow:0 0 0 1px var(--stone-200); color:var(--stone-500); text-transform:uppercase; letter-spacing:0.03em; } + .pill.ok { color:var(--brand-700); box-shadow:0 0 0 1px var(--brand-100); background:var(--brand-50); } + .pill.warn { color:#9a7a2e; box-shadow:0 0 0 1px #ecdcae; background:#faf5e6; } + + code.token { display:block; margin:12px 0; padding:12px 14px; background:var(--stone-50); + box-shadow:0 0 0 1px var(--stone-200); border-radius:8px; font-family:"Geist Mono",monospace; + font-size:13px; word-break:break-all; color:var(--stone-900); } + .notice { margin-top:16px; padding:14px 16px; background:var(--brand-50); border-radius:8px; } + .notice b { color:var(--brand-700); } + .err { color:#b91c1c; font-size:13px; min-height:18px; margin-top:8px; } + .foot { text-align:center; color:var(--stone-400); font-size:12px; margin-top:28px; } + .foot a { color:var(--stone-500); } + a { color:var(--brand-600); text-decoration:none; } + a:hover { text-decoration:underline; } +`; + +function shell(title: string, body: string): string { + return ` + +${title} · CrossPoint Sync + + + + + + +
+ CrossPoint Sync + ${title === 'Account' ? '' : ''} +
+
${body}
+`; +} + +const LANDING = shell( + 'Sign in', + `
+ Sync Hub +

Your reading, everywhere

+

One account to sync reading progress across your devices and link services like Hardcover, Readwise, and BookFusion.

+
+ +
+

Create account

+ + + +
+ +
+ +
+

Sign in

+ + + +
+
+ +

Want to self host this?

+ +` +); + +const SECTION = 'font-size:13px;text-transform:uppercase;letter-spacing:0.05em;color:var(--stone-500);margin:32px 0 12px;'; + +const ACCOUNT = shell( + 'Account', + `
+ Account +

Signed in as …

+
+ +

CrossPoint Sync (KOSync)

+

Loading…

+ +

Linked services

+

Loading…

+ +

Website login

+
+

The token you use to sign into this website. Rotating it signs you out on the web and does not affect your reader.

+
+ + +
+ +
+ +

Want to self host this?

+ +` +); + +const LINK = shell( + 'Link service', + ` +
Link service +

Link

+

+

Loading…

+ +` +); + +export function webRoutes(): Hono { + const app = new Hono(); + + app.get('/logo.png', (c) => { + c.header('content-type', 'image/png'); + c.header('cache-control', 'public, max-age=86400'); + return c.body(LOGO); + }); + app.get('/favicon.png', (c) => { + c.header('content-type', 'image/png'); + c.header('cache-control', 'public, max-age=86400'); + return c.body(FAVICON); + }); + + app.get('/', (c) => { + if (verifySession(getCookie(c, SESSION_COOKIE))) return c.redirect('/account'); + return c.html(LANDING); + }); + + app.get('/account', (c) => { + if (!verifySession(getCookie(c, SESSION_COOKIE))) return c.redirect('/'); + return c.html(ACCOUNT); + }); + + app.get('/link/:id', (c) => { + if (!verifySession(getCookie(c, SESSION_COOKIE))) return c.redirect('/'); + return c.html(LINK); + }); + + return app; +} diff --git a/test/account.test.ts b/test/account.test.ts new file mode 100644 index 0000000..215e461 --- /dev/null +++ b/test/account.test.ts @@ -0,0 +1,219 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { makeTestApp, md5, DOC } from './helpers.js'; +import { resetSessionSecretCache } from '../src/auth/session.js'; + +type App = ReturnType['app']; + +afterEach(() => resetSessionSecretCache()); + +async function signup(app: App, handle: string) { + return app.request('/auth/signup', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ handle }), + }); +} + +async function signupSession(app: App, handle: string): Promise<{ token: string; cookie: string }> { + const res = await signup(app, handle); + const body = await res.json(); + return { token: body.token, cookie: res.headers.get('set-cookie')!.split(';')[0] }; +} + +async function createKosync(app: App, cookie: string, username = 'julia', password = 'reader-pw') { + const res = await app.request('/account/kosync', { + method: 'POST', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ username, password }), + }); + return { status: res.status, data: await res.json() }; +} + +describe('master (website) account', () => { + it('signup issues an xp1_ login token and a session cookie', async () => { + const { app } = makeTestApp(); + const res = await signup(app, 'julia'); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.handle).toBe('julia'); + expect(body.token).toMatch(/^xp1_\d+_[0-9a-f]{32}$/); + const cookie = res.headers.get('set-cookie')!.split(';')[0]; + const me = await app.request('/auth/me', { headers: { cookie } }); + expect((await me.json()).handle).toBe('julia'); + }); + + it('the master token is NOT a kosync device secret', async () => { + const { app } = makeTestApp(); + const { token } = await signupSession(app, 'julia'); + const auth = await app.request('/users/auth', { + headers: { 'x-auth-user': 'julia', 'x-auth-key': md5(token) }, + }); + expect(auth.status).toBe(401); + }); + + it('login with the token establishes a session', async () => { + const { app } = makeTestApp(); + const { token } = await signupSession(app, 'julia'); + const login = await app.request('/auth/login', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ token }), + }); + expect(login.status).toBe(200); + }); + + it('v1 data endpoints return 409 until a kosync account is linked', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + const res = await app.request('/api/v1/connectors', { headers: { cookie } }); + expect(res.status).toBe(409); + }); +}); + +describe('kosync account linked under a master account', () => { + it('create a kosync account with a chosen password; reader + web both work', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + const { status, data } = await createKosync(app, cookie, 'julia', 'reader-pw'); + expect(status).toBe(200); + expect(data.username).toBe('julia'); + expect(data.token).toBeUndefined(); // no auto-generated secret + // The chosen password authenticates the reader (device sends MD5(password)). + const auth = await app.request('/users/auth', { + headers: { 'x-auth-user': 'julia', 'x-auth-key': md5('reader-pw') }, + }); + expect(auth.status).toBe(200); + const conn = await app.request('/api/v1/connectors', { headers: { cookie } }); + expect(conn.status).toBe(200); + }); + + it('rejects create without a password', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + const res = await app.request('/account/kosync', { + method: 'POST', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'julia' }), + }); + expect(res.status).toBe(400); + }); + + it('kosync status reflects linked state', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + expect((await (await app.request('/account/kosync', { headers: { cookie } })).json()).linked).toBe(false); + await createKosync(app, cookie); + const status = await (await app.request('/account/kosync', { headers: { cookie } })).json(); + expect(status).toMatchObject({ linked: true, username: 'julia' }); + }); + + it('link an existing (device-created) kosync account by password', async () => { + const { app } = makeTestApp(); + const password = 'reader-pass'; + await app.request('/users/create', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'reader1', password: md5(password) }), + }); + const { cookie } = await signupSession(app, 'julia'); + const link = await app.request('/account/kosync', { + method: 'PUT', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'reader1', password }), + }); + expect(link.status).toBe(200); + expect((await app.request('/api/v1/connectors', { headers: { cookie } })).status).toBe(200); + }); + + it('rejects linking with a wrong password', async () => { + const { app } = makeTestApp(); + await app.request('/users/create', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'reader1', password: md5('right') }), + }); + const { cookie } = await signupSession(app, 'julia'); + const link = await app.request('/account/kosync', { + method: 'PUT', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'reader1', password: 'wrong' }), + }); + expect(link.status).toBe(401); + }); + + it('change password revokes the old one', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + await createKosync(app, cookie, 'julia', 'oldpw'); + const res = await app.request('/account/kosync/password', { + method: 'POST', + headers: { cookie, 'content-type': 'application/json' }, + body: JSON.stringify({ password: 'newpw' }), + }); + expect(res.status).toBe(200); + expect((await app.request('/users/auth', { headers: { 'x-auth-user': 'julia', 'x-auth-key': md5('newpw') } })).status).toBe(200); + expect((await app.request('/users/auth', { headers: { 'x-auth-user': 'julia', 'x-auth-key': md5('oldpw') } })).status).toBe(401); + }); + + it('device sync data is readable from the web session', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + const { data } = await createKosync(app, cookie, 'julia', 'reader-pw'); + const headers = { 'x-auth-user': data.username, 'x-auth-key': md5('reader-pw'), 'content-type': 'application/json' }; + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.4, device_id: 'd1' }), + }); + const list = await app.request('/api/v1/progress', { headers: { cookie } }); + expect(list.status).toBe(200); + expect((await list.json()).items[0].document).toBe(DOC); + }); + + it('respects REGISTRATION_DISABLED for master signup', async () => { + const { app } = makeTestApp({ registrationDisabled: true }); + expect((await signup(app, 'nope')).status).toBe(403); + }); +}); + +describe('deletion', () => { + it('deletes the kosync account and its data, freeing the username', async () => { + const { app } = makeTestApp(); + const { cookie } = await signupSession(app, 'julia'); + await createKosync(app, cookie, 'julia', 'pw'); + // Push some data. + const headers = { 'x-auth-user': 'julia', 'x-auth-key': md5('pw'), 'content-type': 'application/json' }; + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.4, device_id: 'd1' }), + }); + const del = await app.request('/account/kosync/data', { method: 'DELETE', headers: { cookie } }); + expect(del.status).toBe(200); + // kosync auth no longer works; v1 falls back to 409 (no linked sync). + expect((await app.request('/users/auth', { headers })).status).toBe(401); + expect((await app.request('/api/v1/connectors', { headers: { cookie } })).status).toBe(409); + // Username is free to reuse. + const status = await (await app.request('/account/kosync', { headers: { cookie } })).json(); + expect(status.linked).toBe(false); + }); + + it('deletes the master account, its kosync account, and clears the session', async () => { + const { app } = makeTestApp(); + const { cookie, token } = await signupSession(app, 'julia'); + await createKosync(app, cookie, 'julia', 'pw'); + const del = await app.request('/account', { method: 'DELETE', headers: { cookie } }); + expect(del.status).toBe(200); + // Master login token no longer works. + const login = await app.request('/auth/login', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ token }), + }); + expect(login.status).toBe(401); + // kosync account is gone too. + expect( + (await app.request('/users/auth', { headers: { 'x-auth-user': 'julia', 'x-auth-key': md5('pw') } })).status + ).toBe(401); + }); +}); diff --git a/test/connectors-more.test.ts b/test/connectors-more.test.ts new file mode 100644 index 0000000..7468538 --- /dev/null +++ b/test/connectors-more.test.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { DOC, makeTestApp, registerUser } from './helpers.js'; +import { resetEncryptionKeyCache } from '../src/crypto/secrets.js'; +import type { HttpTransport } from '../src/connectors/types.js'; +import { drainQueue } from '../src/connectors/runner.js'; +import { claimReady } from '../src/connectors/queue.js'; +import { kosyncConnector, baseUrl } from '../src/connectors/kosync.js'; +import { bookfusionConnector, extractBooks } from '../src/connectors/bookfusion.js'; + +function fakeTransport() { + const calls: { url: string; method: string; body?: string }[] = []; + const handlers: { match: string; status: number; body: unknown }[] = []; + const t: HttpTransport = async (url, init) => { + calls.push({ url, method: init.method, body: init.body }); + const h = [...handlers].reverse().find((x) => url.includes(x.match) || (init.body ?? '').includes(x.match)); + const status = h?.status ?? 200; + const body = h?.body ?? {}; + return { status, text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), json: async () => body }; + }; + return { transport: t, calls, on: (m: string, s: number, b: unknown) => handlers.push({ match: m, status: s, body: b }) }; +} + +const KEY = { TOKEN_ENC_KEY: 'a'.repeat(64) }; +beforeEach(() => { Object.assign(process.env, KEY); resetEncryptionKeyCache(); }); +afterEach(() => { delete process.env.TOKEN_ENC_KEY; resetEncryptionKeyCache(); }); + +describe('kosync mirror connector (unit)', () => { + it('normalizes server URLs', () => { + expect(baseUrl('sync.koreader.rocks:443')).toBe('https://sync.koreader.rocks:443'); + expect(baseUrl('https://x.com/')).toBe('https://x.com'); + }); + + it('matches by identity (no network, same document hash)', async () => { + const fake = fakeTransport(); + const m = await kosyncConnector.match({ server: 's', username: 'u', password: 'p' }, { document: DOC, title: null, author: null, filename: null }, fake.transport); + expect(m).toEqual({ externalId: DOC, confidence: 1 }); + expect(fake.calls).toHaveLength(0); + }); + + it('validate hits /users/auth with x-auth headers', async () => { + const fake = fakeTransport(); + fake.on('/users/auth', 200, {}); + const v = await kosyncConnector.validate({ server: 'srv.test', username: 'u', password: 'p' }, fake.transport); + expect(v.ok).toBe(true); + expect(fake.calls[0].url).toContain('/users/auth'); + }); + + it('push forwards progress string + position to the target', async () => { + const fake = fakeTransport(); + fake.on('/syncs/progress', 200, {}); + const r = await kosyncConnector.push( + { server: 'srv.test', username: 'u', password: 'p' }, + { externalId: DOC, confidence: 1 }, + { kind: 'progress', document: DOC, percentage: 0.4, progress: '/body/p[1]', position: { pctQ: 400000 }, timestamp: 1 }, + fake.transport + ); + expect(r.ok).toBe(true); + const body = JSON.parse(fake.calls[0].body!); + expect(body).toMatchObject({ document: DOC, progress: '/body/p[1]', percentage: 0.4, position: { pctQ: 400000 } }); + }); +}); + +describe('kosync mirror fan-out (end to end)', () => { + it('a device progress push enqueues + delivers to the mirror', async () => { + const fake = fakeTransport(); + fake.on('/users/auth', 200, {}); // validate on link + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + // Link an external kosync mirror. + const link = await app.request('/api/v1/connectors/kosync', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { server: 'mirror.test', username: 'u', password: 'p' } }), + }); + expect(link.status).toBe(200); + // Device pushes progress. + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: '/body/p[2]', percentage: 0.5, device_id: 'd1' }), + }); + expect(claimReady(db, 10).length).toBeGreaterThan(0); + fake.on('/syncs/progress', 200, {}); + await drainQueue(db, fake.transport, 10); + // A mirror PUT to the target server happened. + expect(fake.calls.some((c) => c.url.includes('mirror.test') && c.url.includes('/syncs/progress') && c.method === 'PUT')).toBe(true); + expect(claimReady(db, 10)).toHaveLength(0); + }); +}); + +describe('bookfusion connector', () => { + it('extractBooks handles the search payload', () => { + const hits = extractBooks({ books: [{ id: 7, title: 'Foundryside', authors: [{ name: 'Robert Jackson Bennett' }] }] }); + expect(hits).toEqual([{ externalId: '7', title: 'Foundryside', author: 'Robert Jackson Bennett' }]); + }); + + it('device-code begin + poll yields a credential', async () => { + const fake = fakeTransport(); + fake.on('/api/user/auth/device', 200, { device_code: 'DC', user_code: 'WXYZ', verification_uri: 'https://bookfusion.com/link', interval: 1, expires_in: 900 }); + const start = await bookfusionConnector.beginLink!(fake.transport); + expect(start).toMatchObject({ deviceCode: 'DC', userCode: 'WXYZ' }); + + fake.on('/api/user/auth/token', 200, { error: 'authorization_pending' }); + expect((await bookfusionConnector.pollLink!('DC', fake.transport)).status).toBe('pending'); + fake.on('/api/user/auth/token', 200, { access_token: 'BF-TOKEN' }); + const done = await bookfusionConnector.pollLink!('DC', fake.transport); + expect(done.status).toBe('ok'); + expect(done.credential).toEqual({ access_token: 'BF-TOKEN' }); + }); + + it('link/begin + link/poll endpoints link the account', async () => { + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + fake.on('/api/user/auth/device', 200, { device_code: 'DC', user_code: 'WXYZ', verification_uri: 'https://bookfusion.com/link' }); + const begin = await app.request('/api/v1/connectors/bookfusion/link/begin', { method: 'POST', headers }); + expect(begin.status).toBe(200); + const { device_code } = await begin.json(); + + fake.on('/api/user/auth/token', 200, { access_token: 'BF-TOKEN' }); + fake.on('/api/user/books/search', 200, {}); // validate + const poll = await app.request('/api/v1/connectors/bookfusion/link/poll', { + method: 'POST', + headers, + body: JSON.stringify({ device_code }), + }); + expect(poll.status).toBe(200); + expect((await poll.json()).linked).toBe(true); + + const list = await (await app.request('/api/v1/connectors', { headers })).json(); + expect(list.connectors.find((c: { id: string }) => c.id === 'bookfusion').linked).toBe(true); + }); + + it('push maps 0..1 to 0..100 reading_position', async () => { + const fake = fakeTransport(); + fake.on('/reading_position', 200, {}); + const r = await bookfusionConnector.push( + { access_token: 't' }, + { externalId: '7', confidence: 1 }, + { kind: 'progress', document: DOC, percentage: 0.25, timestamp: 1 }, + fake.transport + ); + expect(r.ok).toBe(true); + expect(JSON.parse(fake.calls[0].body!).percentage).toBe(25); + }); +}); diff --git a/test/connectors.test.ts b/test/connectors.test.ts new file mode 100644 index 0000000..ea7d367 --- /dev/null +++ b/test/connectors.test.ts @@ -0,0 +1,237 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { DOC, makeTestApp, registerUser } from './helpers.js'; +import { resetEncryptionKeyCache } from '../src/crypto/secrets.js'; +import type { HttpTransport } from '../src/connectors/types.js'; +import { drainQueue } from '../src/connectors/runner.js'; +import { claimReady } from '../src/connectors/queue.js'; + +// A programmable fake transport. Records requests; returns queued responses by +// URL substring match. +function fakeTransport() { + const calls: { url: string; method: string; body?: string }[] = []; + const handlers: { match: string; status: number; body: unknown }[] = []; + const t: HttpTransport = async (url, init) => { + calls.push({ url, method: init.method, body: init.body }); + // Match on URL or request body; most-recently-registered wins so specific + // GraphQL operations (Search, mutations) override a broad 'graphql' handler. + const h = [...handlers] + .reverse() + .find((x) => url.includes(x.match) || (init.body ?? '').includes(x.match)); + const status = h?.status ?? 200; + const body = h?.body ?? {}; + return { + status, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + json: async () => body, + }; + }; + return { + transport: t, + calls, + on(match: string, status: number, body: unknown) { + handlers.push({ match, status, body }); + }, + }; +} + +const KEY = { TOKEN_ENC_KEY: 'a'.repeat(64) }; + +beforeEach(() => { + Object.assign(process.env, KEY); + resetEncryptionKeyCache(); +}); +afterEach(() => { + delete process.env.TOKEN_ENC_KEY; + resetEncryptionKeyCache(); +}); + +describe('connector management API', () => { + it('lists connectors with encryption status and unlinked state', async () => { + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors', { headers }); + const body = await res.json(); + expect(body.encryption).toBe('enabled'); + const ids = body.connectors.map((c: { id: string }) => c.id).sort(); + // Readwise is hidden for now; still registered but not listed. + expect(ids).toEqual(['bookfusion', 'hardcover', 'kosync']); + expect(body.connectors.every((c: { linked: boolean }) => !c.linked)).toBe(true); + }); + + it('rejects linking when TOKEN_ENC_KEY is unset', async () => { + delete process.env.TOKEN_ENC_KEY; + resetEncryptionKeyCache(); + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'x' } }), + }); + expect(res.status).toBe(403); + }); + + it('validates and links Hardcover, then reports linked', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const link = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc-token' } }), + }); + expect(link.status).toBe(200); + expect((await link.json()).account).toBe('julia'); + + const list = await (await app.request('/api/v1/connectors', { headers })).json(); + const hc = list.connectors.find((c: { id: string }) => c.id === 'hardcover'); + expect(hc.linked).toBe(true); + expect(hc.account).toBe('julia'); + }); + + it('rejects an invalid credential (validate fails)', async () => { + const fake = fakeTransport(); + fake.on('graphql', 401, {}); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + const res = await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'bad' } }), + }); + expect(res.status).toBe(400); + }); + + it('unlink wipes account, matches, and queue', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + const del = await app.request('/api/v1/connectors/hardcover', { method: 'DELETE', headers }); + expect(del.status).toBe(200); + const list = await (await app.request('/api/v1/connectors', { headers })).json(); + expect(list.connectors.find((c: { id: string }) => c.id === 'hardcover').linked).toBe(false); + }); +}); + +describe('fan-out on progress sync', () => { + it('enqueues a progress event for a linked write connector and pushes it', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + + // Link Hardcover. + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + // Provide metadata so matching can work, then sync progress. + await app.request('/api/v1/documents', { + method: 'PUT', + headers, + body: JSON.stringify({ + items: [{ document: DOC, title: 'Foundryside', author: 'Robert Jackson Bennett' }], + }), + }); + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.3, device_id: 'd1' }), + }); + + // A pending queue row should exist. + expect(claimReady(db, 10).length).toBeGreaterThan(0); + + // Now the drain: match (search) then push (mutation). + fake.on('Search', 200, { + data: { search: { results: [{ document: { id: 42, title: 'Foundryside', author_names: ['Robert Jackson Bennett'] } }] } }, + }); + // The push mutation returns success. + fake.on('insert_user_book', 200, { data: { insert_user_book: { id: 1 } } }); + + await drainQueue(db, fake.transport, 10); + + // Queue drained. + expect(claimReady(db, 10)).toHaveLength(0); + // A mutation call was made. + expect(fake.calls.some((c) => c.body?.includes('insert_user_book'))).toBe(true); + }); + + it('does not fan out when no connector is linked', async () => { + const fake = fakeTransport(); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/syncs/progress', { + method: 'PUT', + headers, + body: JSON.stringify({ document: DOC, progress: 'p', percentage: 0.3, device_id: 'd1' }), + }); + expect(claimReady(db, 10)).toHaveLength(0); + }); + + it('manual match override is honored and sticky', async () => { + const fake = fakeTransport(); + fake.on('graphql', 200, { data: { me: [{ username: 'julia' }] } }); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/hardcover', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'hc' } }), + }); + const set = await app.request(`/api/v1/connectors/hardcover/matches/${DOC}`, { + method: 'PUT', + headers, + body: JSON.stringify({ external_id: '999' }), + }); + expect(set.status).toBe(200); + const list = await ( + await app.request('/api/v1/connectors/hardcover/matches', { headers }) + ).json(); + expect(list.matches[0]).toMatchObject({ document: DOC, external_id: '999', source: 'manual' }); + }); +}); + +describe('readwise highlight fan-out', () => { + it('pushes a clipping as a highlight', async () => { + const fake = fakeTransport(); + fake.on('/auth/', 204, {}); + const { app, db } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + await app.request('/api/v1/connectors/readwise', { + method: 'PUT', + headers, + body: JSON.stringify({ credential: { token: 'rw' } }), + }); + await app.request('/api/v1/documents', { + method: 'PUT', + headers, + body: JSON.stringify({ items: [{ document: DOC, title: 'Foundryside', author: 'RJB' }] }), + }); + await app.request(`/api/v1/clippings/${DOC}`, { + method: 'PUT', + headers, + body: JSON.stringify({ + items: [ + { id: 'c0ffee0011223344', spine: 1, text: 'a memorable line', created_at: 1752300000 }, + ], + }), + }); + expect(claimReady(db, 10).length).toBeGreaterThan(0); + + fake.on('/highlights/', 200, [{ id: 1 }]); + await drainQueue(db, fake.transport, 10); + expect(fake.calls.some((c) => c.url.includes('/highlights/') && c.method === 'POST')).toBe(true); + expect(claimReady(db, 10)).toHaveLength(0); + }); +}); diff --git a/test/helpers.ts b/test/helpers.ts index d804f86..e1dde7c 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import type { Hono } from 'hono'; -import { createApp } from '../src/app.js'; +import { createApp, type AppOptions } from '../src/app.js'; import type { Config } from '../src/config.js'; import { migrate, openDatabase, type DB } from '../src/db/db.js'; import type { AppEnv } from '../src/auth/middleware.js'; @@ -14,7 +14,10 @@ export interface TestServer { db: DB; } -export function makeTestApp(configOverrides: Partial = {}): TestServer { +export function makeTestApp( + configOverrides: Partial = {}, + opts: AppOptions = {} +): TestServer { const db = openDatabase(':memory:'); migrate(db); const config: Config = { @@ -22,7 +25,7 @@ export function makeTestApp(configOverrides: Partial = {}): TestServer { authRateLimitPerMinute: 0, // disabled in tests (limiter state is per-app anyway) ...configOverrides, }; - return { app: createApp(db, config), db }; + return { app: createApp(db, config, opts), db }; } let userCounter = 0; diff --git a/test/matching.test.ts b/test/matching.test.ts new file mode 100644 index 0000000..d8ed089 --- /dev/null +++ b/test/matching.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; +import { + coreTitle, + decideMatch, + extractTitleAuthor, + normalizeAuthor, + scoreCandidate, +} from '../src/connectors/matching.js'; + +describe('matching helpers', () => { + it('strips subtitles and series suffixes from titles', () => { + expect(coreTitle('Foundryside: A Novel')).toBe('Foundryside'); + expect(coreTitle('Foundryside (The Founders Trilogy, Book 1)')).toBe('Foundryside'); + expect(coreTitle('Plain Title')).toBe('Plain Title'); + }); + + it('normalizes "Last, First" author form', () => { + expect(normalizeAuthor('Bennett, Robert Jackson')).toBe('robert jackson bennett'); + expect(normalizeAuthor('Robert Jackson Bennett')).toBe('robert jackson bennett'); + }); + + it('uses EPUB title/author as the primary signal', () => { + expect(extractTitleAuthor({ document: 'd', title: 'Foundryside', author: 'RJB', filename: 'x.epub' })) + .toEqual({ title: 'Foundryside', author: 'RJB' }); + }); + + it('falls back to a filename query only when title is missing, without guessing order', () => { + expect( + extractTitleAuthor({ + document: 'd', + title: null, + author: null, + filename: 'Foundryside - Robert Jackson Bennett.epub', + }) + ).toEqual({ title: 'Foundryside Robert Jackson Bennett', author: '' }); + expect(extractTitleAuthor({ document: 'd', title: null, author: null, filename: null })).toBeNull(); + }); + + it('scores exact title+author near 1 and mismatches low', () => { + const good = scoreCandidate('Foundryside', 'Robert Jackson Bennett', { + externalId: '1', + title: 'Foundryside', + author: 'Robert Jackson Bennett', + }); + expect(good).toBeGreaterThan(0.9); + const bad = scoreCandidate('Foundryside', 'Robert Jackson Bennett', { + externalId: '2', + title: 'Dune', + author: 'Frank Herbert', + }); + expect(bad).toBeLessThan(0.2); + }); + + it('auto-accepts a clear winner, rejects ambiguous title collisions', () => { + const clear = decideMatch('Foundryside', 'Robert Jackson Bennett', [ + { externalId: '1', title: 'Foundryside', author: 'Robert Jackson Bennett' }, + { externalId: '2', title: 'Dune', author: 'Frank Herbert' }, + ]); + expect(clear.accepted).toBe(true); + expect(clear.best?.externalId).toBe('1'); + + // Two different books literally titled "Circe" — needs author to split; with + // no author on the query, stays unaccepted. + const collision = decideMatch('Circe', '', [ + { externalId: '1', title: 'Circe', author: 'Madeline Miller' }, + { externalId: '2', title: 'Circe', author: 'Someone Else' }, + ]); + expect(collision.accepted).toBe(false); + }); + + it('accepts same-book different-edition ambiguity, taking the popular one', () => { + const d = decideMatch('Foundryside', 'Robert Jackson Bennett', [ + { externalId: 'ed1', title: 'Foundryside', author: 'Robert Jackson Bennett', popularity: 10 }, + { externalId: 'ed2', title: 'Foundryside', author: 'Robert Jackson Bennett', popularity: 500 }, + ]); + expect(d.accepted).toBe(true); + expect(d.best?.externalId).toBe('ed2'); + }); + + it('returns unaccepted for empty candidates', () => { + expect(decideMatch('X', 'Y', []).accepted).toBe(false); + }); +}); diff --git a/test/queue.test.ts b/test/queue.test.ts new file mode 100644 index 0000000..eccc9d1 --- /dev/null +++ b/test/queue.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import { migrate, openDatabase, type DB } from '../src/db/db.js'; +import { + claimReady, + enqueue, + markDone, + markFailed, + queueDepth, + _internals, +} from '../src/connectors/queue.js'; +import type { OutboundEvent } from '../src/connectors/types.js'; + +function freshDb(): DB { + const db = openDatabase(':memory:'); + migrate(db); + db.prepare('INSERT INTO users (id, username, key_hash, created_at) VALUES (1, ?, ?, 0)').run( + 'u', + 'h' + ); + return db; +} + +const ev = (over: Partial = {}): OutboundEvent => ({ + kind: 'progress', + document: 'a'.repeat(32), + percentage: 0.5, + timestamp: 1000, + ...over, +}); + +describe('connector queue', () => { + it('coalesces same (connector, document, kind) to the latest payload', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev({ percentage: 0.3 }), 'progress', 100); + enqueue(db, 1, 'hardcover', ev({ percentage: 0.6 }), 'progress', 101); + const ready = claimReady(db, 10, 200); + expect(ready).toHaveLength(1); + expect(JSON.parse(ready[0].payload).percentage).toBe(0.6); + }); + + it('keeps distinct highlight coalesce keys separate', () => { + const db = freshDb(); + enqueue(db, 1, 'readwise', ev({ kind: 'highlight' }), 'highlight:c1', 100); + enqueue(db, 1, 'readwise', ev({ kind: 'highlight' }), 'highlight:c2', 100); + expect(claimReady(db, 10, 200)).toHaveLength(2); + }); + + it('only returns rows whose next_try_at has passed', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 500); + expect(claimReady(db, 10, 400)).toHaveLength(0); + expect(claimReady(db, 10, 500)).toHaveLength(1); + }); + + it('markDone removes the row from the ready set', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 100); + const [row] = claimReady(db, 10, 200); + markDone(db, row.id, 200); + expect(claimReady(db, 10, 300)).toHaveLength(0); + expect(queueDepth(db, 1, 'hardcover')).toEqual({ pending: 0, dead: 0 }); + }); + + it('retryable failure backs off; non-retryable dead-letters immediately', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 100); + let [row] = claimReady(db, 10, 100); + markFailed(db, row, 'boom', true, 100); + // Not ready immediately after backoff window start + expect(claimReady(db, 10, 100)).toHaveLength(0); + expect(claimReady(db, 10, 100 + _internals.BACKOFF[0])).toHaveLength(1); + + enqueue(db, 1, 'readwise', ev(), 'highlight:x', 100); + [row] = claimReady(db, 10, 100).filter((r) => r.connector_id === 'readwise'); + markFailed(db, row, 'permanent', false, 100); + expect(queueDepth(db, 1, 'readwise')).toEqual({ pending: 0, dead: 1 }); + }); + + it('dead-letters after MAX_ATTEMPTS retryable failures', () => { + const db = freshDb(); + enqueue(db, 1, 'hardcover', ev(), 'progress', 0); + let t = 0; + for (let i = 0; i < _internals.MAX_ATTEMPTS; i++) { + const ready = claimReady(db, 10, t); + expect(ready).toHaveLength(1); + markFailed(db, ready[0], 'again', true, t); + t += 1_000_000; // jump past any backoff + } + expect(queueDepth(db, 1, 'hardcover')).toEqual({ pending: 0, dead: 1 }); + }); +}); diff --git a/test/secrets.test.ts b/test/secrets.test.ts new file mode 100644 index 0000000..8ac49b0 --- /dev/null +++ b/test/secrets.test.ts @@ -0,0 +1,54 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { + decryptSecret, + encryptSecret, + getEncryptionKey, + resetEncryptionKeyCache, + secretsEnabled, +} from '../src/crypto/secrets.js'; + +const KEY_HEX = 'a'.repeat(64); + +afterEach(() => resetEncryptionKeyCache()); + +describe('secret vault', () => { + it('round-trips a token', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + const enc = encryptSecret('hardcover-token-xyz', env); + expect(enc).not.toContain('hardcover-token-xyz'); + expect(decryptSecret(enc, env)).toBe('hardcover-token-xyz'); + }); + + it('produces different ciphertext each call (random IV)', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + expect(encryptSecret('same', env)).not.toBe(encryptSecret('same', env)); + }); + + it('detects tampering via the auth tag', () => { + const env = { TOKEN_ENC_KEY: KEY_HEX } as NodeJS.ProcessEnv; + const enc = encryptSecret('secret', env); + const buf = Buffer.from(enc, 'base64'); + buf[buf.length - 1] ^= 0xff; // flip a ciphertext byte + expect(() => decryptSecret(buf.toString('base64'), env)).toThrow(); + }); + + it('accepts a base64 32-byte key and a passphrase', () => { + const b64 = { TOKEN_ENC_KEY: Buffer.alloc(32, 7).toString('base64') } as NodeJS.ProcessEnv; + expect(decryptSecret(encryptSecret('x', b64), b64)).toBe('x'); + resetEncryptionKeyCache(); + const phrase = { TOKEN_ENC_KEY: 'this is a long enough passphrase!!' } as NodeJS.ProcessEnv; + expect(decryptSecret(encryptSecret('y', phrase), phrase)).toBe('y'); + }); + + it('is disabled when the key is unset', () => { + const env = {} as NodeJS.ProcessEnv; + expect(secretsEnabled(env)).toBe(false); + expect(getEncryptionKey(env)).toBeNull(); + expect(() => encryptSecret('x', env)).toThrow(/TOKEN_ENC_KEY/); + }); + + it('rejects an obviously too-short key', () => { + const env = { TOKEN_ENC_KEY: 'short' } as NodeJS.ProcessEnv; + expect(() => getEncryptionKey(env)).toThrow(); + }); +});