diff --git a/harnesses/openclaw/src/index.ts b/harnesses/openclaw/src/index.ts index 23363c70..91de87eb 100644 --- a/harnesses/openclaw/src/index.ts +++ b/harnesses/openclaw/src/index.ts @@ -66,6 +66,7 @@ import { readVirtualPathContent } from "../../../src/hooks/virtual-table-query.j // message_embedding (today's behavior, preserved on every failure mode). import { tryEmbedStandalone, _setSpawnImpl } from "../../../src/embeddings/standalone-embed-client.js"; import { embeddingSqlLiteral } from "../../../src/embeddings/sql.js"; +import { redactSecrets } from "../../../src/hooks/shared/redact.js"; // Resolve sibling skillify-worker.js path at runtime via import.meta.url. The // openclaw plugin is bundled to harnesses/openclaw/dist/index.js, then installed to // ~/.openclaw/extensions/hivemind/dist/index.js by install-openclaw.ts. The @@ -1521,7 +1522,8 @@ export default definePluginEntry({ content: text, timestamp: ts, }; - const line = JSON.stringify(entry); + // Mask secrets before the payload is embedded or stored. + const line = redactSecrets(JSON.stringify(entry)); // For JSONB: only escape single quotes, keep JSON structure intact const jsonForSql = line.replace(/'/g, "''"); diff --git a/src/hooks/capture.ts b/src/hooks/capture.ts index 9f8768fe..dece230c 100644 --- a/src/hooks/capture.ts +++ b/src/hooks/capture.ts @@ -10,6 +10,7 @@ import { readStdin } from "../utils/stdin.js"; import { type Config } from "../config.js"; import { resolveCaptureConfig } from "./shared/dir-gate.js"; +import { redactSecrets } from "./shared/redact.js"; import { DeeplakeApi } from "../deeplake-api.js"; import { sqlStr } from "../utils/sql.js"; import { projectNameFromCwd } from "../utils/project-name.js"; @@ -145,7 +146,10 @@ async function main(): Promise { } const sessionPath = buildSessionPath(config, input.session_id); - const line = JSON.stringify(entry); + // Mask secrets (tokens, passwords, API keys) before the payload is embedded + // or written to the store. Redacting the serialized line covers every field + // (content / tool_input / tool_response) and both egress paths at once. + const line = redactSecrets(JSON.stringify(entry)); log(`writing to ${sessionPath}`); // Simple INSERT — one row per event, no concat, no race conditions. diff --git a/src/hooks/codex/capture.ts b/src/hooks/codex/capture.ts index 824f03a9..f6b33512 100644 --- a/src/hooks/codex/capture.ts +++ b/src/hooks/codex/capture.ts @@ -15,6 +15,7 @@ import { readStdin } from "../../utils/stdin.js"; import { type Config } from "../../config.js"; import { resolveCaptureConfig } from "../shared/dir-gate.js"; +import { redactSecrets } from "../shared/redact.js"; import { DeeplakeApi } from "../../deeplake-api.js"; import { sqlStr } from "../../utils/sql.js"; import { projectNameFromCwd } from "../../utils/project-name.js"; @@ -120,7 +121,7 @@ async function main(): Promise { } const sessionPath = buildSessionPath(config, input.session_id); - const line = JSON.stringify(entry); + const line = redactSecrets(JSON.stringify(entry)); log(`writing to ${sessionPath}`); const projectName = projectNameFromCwd(input.cwd); diff --git a/src/hooks/codex/wiki-worker.ts b/src/hooks/codex/wiki-worker.ts index 4f978bec..3175c73c 100644 --- a/src/hooks/codex/wiki-worker.ts +++ b/src/hooks/codex/wiki-worker.ts @@ -14,6 +14,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; import { EmbedClient } from "../../embeddings/client.js"; @@ -265,7 +266,7 @@ async function main(): Promise { if (raw.trim()) { // Stamp the offset ourselves so the persisted summary is authoritative // and never depends on the LLM echoing the bookkeeping line. - const text = stampOffset(raw, jsonlLines); + const text = redactSecrets(stampOffset(raw, jsonlLines)); const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; // Embed the summary so it ranks in the semantic retrieval branch. diff --git a/src/hooks/cursor/capture.ts b/src/hooks/cursor/capture.ts index cd7c5836..dfdda3a5 100644 --- a/src/hooks/cursor/capture.ts +++ b/src/hooks/cursor/capture.ts @@ -15,6 +15,7 @@ import { readStdin } from "../../utils/stdin.js"; import { resolveCaptureConfig } from "../shared/dir-gate.js"; +import { redactSecrets } from "../shared/redact.js"; import { DeeplakeApi } from "../../deeplake-api.js"; import { sqlStr } from "../../utils/sql.js"; import { projectNameFromCwd } from "../../utils/project-name.js"; @@ -145,7 +146,7 @@ async function main(): Promise { } const sessionPath = buildSessionPath(config, sessionId); - const line = JSON.stringify(entry); + const line = redactSecrets(JSON.stringify(entry)); log(`writing to ${sessionPath}`); const projectName = projectNameFromCwd(cwd); diff --git a/src/hooks/cursor/wiki-worker.ts b/src/hooks/cursor/wiki-worker.ts index a80c516b..1dbbd028 100644 --- a/src/hooks/cursor/wiki-worker.ts +++ b/src/hooks/cursor/wiki-worker.ts @@ -19,6 +19,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; import { EmbedClient } from "../../embeddings/client.js"; @@ -251,7 +252,7 @@ async function main(): Promise { if (raw.trim()) { // Stamp the offset ourselves so the persisted summary is authoritative // and never depends on the LLM echoing the bookkeeping line. - const text = stampOffset(raw, jsonlLines); + const text = redactSecrets(stampOffset(raw, jsonlLines)); const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; // Embed the summary so it ranks in the semantic retrieval branch. diff --git a/src/hooks/hermes/capture.ts b/src/hooks/hermes/capture.ts index 836b22f4..0190d4f6 100644 --- a/src/hooks/hermes/capture.ts +++ b/src/hooks/hermes/capture.ts @@ -16,6 +16,7 @@ import { readStdin } from "../../utils/stdin.js"; import { resolveCaptureConfig } from "../shared/dir-gate.js"; +import { redactSecrets } from "../shared/redact.js"; import { DeeplakeApi } from "../../deeplake-api.js"; import { sqlStr } from "../../utils/sql.js"; import { projectNameFromCwd } from "../../utils/project-name.js"; @@ -129,7 +130,7 @@ async function main(): Promise { } const sessionPath = buildSessionPath(config, sessionId); - const line = JSON.stringify(entry); + const line = redactSecrets(JSON.stringify(entry)); log(`writing to ${sessionPath}`); const projectName = projectNameFromCwd(cwd); diff --git a/src/hooks/hermes/wiki-worker.ts b/src/hooks/hermes/wiki-worker.ts index 23512afb..7a972bae 100644 --- a/src/hooks/hermes/wiki-worker.ts +++ b/src/hooks/hermes/wiki-worker.ts @@ -18,6 +18,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; import { EmbedClient } from "../../embeddings/client.js"; @@ -258,7 +259,7 @@ async function main(): Promise { if (raw.trim()) { // Stamp the offset ourselves so the persisted summary is authoritative // and never depends on the LLM echoing the bookkeeping line. - const text = stampOffset(raw, jsonlLines); + const text = redactSecrets(stampOffset(raw, jsonlLines)); const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; // Embed the summary so it ranks in the semantic retrieval branch. diff --git a/src/hooks/pi/wiki-worker.ts b/src/hooks/pi/wiki-worker.ts index fbb8c822..23138170 100644 --- a/src/hooks/pi/wiki-worker.ts +++ b/src/hooks/pi/wiki-worker.ts @@ -23,6 +23,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; import { EmbedClient } from "../../embeddings/client.js"; @@ -257,7 +258,7 @@ async function main(): Promise { if (raw.trim()) { // Stamp the offset ourselves so the persisted summary is authoritative // and never depends on the LLM echoing the bookkeeping line. - const text = stampOffset(raw, jsonlLines); + const text = redactSecrets(stampOffset(raw, jsonlLines)); const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; // Embed the summary so it ranks in the semantic retrieval branch. diff --git a/src/hooks/shared/redact.ts b/src/hooks/shared/redact.ts new file mode 100644 index 00000000..ec966204 --- /dev/null +++ b/src/hooks/shared/redact.ts @@ -0,0 +1,210 @@ +/** + * Secret redaction for captured session content. + * + * Hivemind captures prompts, tool inputs, tool outputs and assistant messages + * and persists/embeds them. Those payloads routinely contain credentials — + * a Bash `export GITHUB_TOKEN=…`, an `Authorization: Bearer …` header echoed + * by curl, a connection string with an inline password, or an API response + * carrying a fresh access token. We mask those with stars BEFORE the text is + * embedded or written to the store, so a leaked secret never lands in the + * vector or the row. + * + * Coverage is layered, most-specific first: + * 1. Private-key blocks (PEM / OpenSSH). + * 2. ~30 known provider token schemes (keep the scheme prefix as a hint). + * 3. Structured secrets: Authorization headers, bare Bearer, URL basic-auth, + * Sentry DSN, Slack webhooks. + * 4. Generic `KEY=VALUE` / `"key":"value"` where KEY is a secret-ish word. + * 5. High-entropy backstop: a bare, unlabeled, random-looking token with no + * known prefix (e.g. an AWS secret access key, a raw API key in JSON). + * + * Design: + * - Fixed-length mask (`MASK`) — never reveal the secret's length. + * - Keep a NON-secret hint where useful: the scheme prefix of a known token + * (`ghp_********`) or the key name of an assignment (`password=********`). + * - Precision guards on the generic/entropy layers so we don't mask + * look-alikes (`tokenizer`, `max_tokens`), UUIDs, git SHAs, hashes, or + * literal `true/false/null`. + * - Pure, dependency-free, idempotent, and star-safe: replacements introduce + * only `*` and kept literal prefixes, so running this over a serialized + * JSON string keeps the JSON parseable. + */ + +const MASK = "********"; + +interface Rule { + re: RegExp; + replace: string | ((match: string, ...groups: string[]) => string); +} + +/** Shannon entropy in bits per character. */ +function shannonEntropy(s: string): number { + const freq = new Map(); + for (const ch of s) freq.set(ch, (freq.get(ch) ?? 0) + 1); + let h = 0; + for (const count of freq.values()) { + const p = count / s.length; + h -= p * Math.log2(p); + } + return h; +} + +/** + * Does a bare token look like a random secret? Deliberately conservative so + * the entropy backstop doesn't shred ordinary identifiers in a trace. + */ +function looksLikeSecret(tok: string): boolean { + if (tok.length < 24) return false; + if (/^\d+$/.test(tok)) return false; // pure number + if (/^[0-9a-f]+$/i.test(tok)) return false; // hex hash / git SHA / md5 + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(tok)) return false; // UUID + // Require a mix of character classes — random keys blend cases/digits, while + // English words, dotted versions and snake_case identifiers do not. + const classes = + (/[a-z]/.test(tok) ? 1 : 0) + (/[A-Z]/.test(tok) ? 1 : 0) + (/[0-9]/.test(tok) ? 1 : 0); + if (classes < 2) return false; + return shannonEntropy(tok) >= 3.5; +} + +// Secret-ish key names for the generic assignment rule. Ordered longest-first. +const SECRET_KEY_WORDS = [ + "aws[_-]?secret[_-]?access[_-]?key", + "secret[_-]?access[_-]?key", + "client[_-]?secret", + "access[_-]?key[_-]?id", + "encryption[_-]?key", + "connection[_-]?string", + "private[_-]?key", + "secret[_-]?key", + "access[_-]?key", + "auth[_-]?token", + "refresh[_-]?token", + "access[_-]?token", + "session[_-]?key", + "account[_-]?key", + "id[_-]?token", + "api[_-]?key", + "app[_-]?key", + "pgpassword", + "passphrase", + "password", + "passwd", + "credentials?", + "signature", + "secret", + "token", + "apikey", +].join("|"); + +// Values that are never secrets — keep configs like `secret: false` readable. +const NON_SECRET_VALUE = /^(true|false|null|none|undefined|nil|""|''|\{\}|\[\])$/i; + +const RULES: Rule[] = [ + // ── 1. Private key blocks ──────────────────────────────────────────────── + { + re: /(-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----)[\s\S]*?(-----END [A-Z0-9 ]*PRIVATE KEY-----)/g, + replace: `$1${MASK}$2`, + }, + + // ── 2. Known provider token schemes — keep the scheme prefix ───────────── + // GitHub (classic / oauth / user / server / refresh, fine-grained PAT). + { re: /(github_pat_)[A-Za-z0-9_]{20,}/g, replace: `$1${MASK}` }, + { re: /(gh[pousr]_)[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + // OpenAI / Anthropic (sk-, sk-ant-, sk-proj-). + { re: /(sk-(?:ant-|proj-)?)[A-Za-z0-9_-]{16,}/g, replace: `$1${MASK}` }, + // Stripe secret / restricted / webhook. + { re: /((?:sk|rk)_(?:live|test)_)[A-Za-z0-9]{16,}/g, replace: `$1${MASK}` }, + { re: /(whsec_)[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + // Slack (bot/user/app tokens). + { re: /(xox[baprse]-)[A-Za-z0-9-]{8,}/g, replace: `$1${MASK}` }, + { re: /(xapp-)[A-Za-z0-9-]{8,}/g, replace: `$1${MASK}` }, + // AWS access key ids (the secret access key is caught by the entropy layer). + { re: /((?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[A-Z0-9]{16})/g, replace: (_m, p1) => `${(p1 as string).slice(0, 4)}${MASK}` }, + // Google API key + OAuth access token. + { re: /(AIza)[A-Za-z0-9_-]{30,}/g, replace: `$1${MASK}` }, + { re: /(ya29\.)[A-Za-z0-9_-]{20,}/g, replace: `$1${MASK}` }, + // HuggingFace, GitLab, npm, PyPI. + { re: /(hf_)[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + { re: /(glpat-)[A-Za-z0-9_-]{18,}/g, replace: `$1${MASK}` }, + { re: /(npm_)[A-Za-z0-9]{30,}/g, replace: `$1${MASK}` }, + { re: /(pypi-)[A-Za-z0-9_-]{40,}/g, replace: `$1${MASK}` }, + // Shopify, DigitalOcean, Doppler, Databricks, Linear, Postman. + { re: /(shp(?:at|ss|ca|pa)_)[a-fA-F0-9]{20,}/g, replace: `$1${MASK}` }, + { re: /((?:dop|doo|dor)_v1_)[a-f0-9]{40,}/g, replace: `$1${MASK}` }, + { re: /(dp\.pt\.)[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + { re: /(dapi)[a-f0-9]{28,}/g, replace: `$1${MASK}` }, + { re: /(lin_api_)[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + { re: /(PMAK-)[a-f0-9]{20,}-[a-f0-9]{20,}/g, replace: `$1${MASK}` }, + // SendGrid, Atlassian, Notion, Groq, xAI, Replicate, Mailgun, Telegram bot. + { re: /(SG\.)[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/g, replace: `$1${MASK}` }, + { re: /(ATATT)[A-Za-z0-9_\-=]{20,}/g, replace: `$1${MASK}` }, + { re: /((?:secret_|ntn_))[A-Za-z0-9]{40,}/g, replace: `$1${MASK}` }, + { re: /(gsk_)[A-Za-z0-9]{40,}/g, replace: `$1${MASK}` }, + { re: /(xai-)[A-Za-z0-9]{60,}/g, replace: `$1${MASK}` }, + { re: /(r8_)[A-Za-z0-9]{30,}/g, replace: `$1${MASK}` }, + { re: /(key-)[a-f0-9]{32}/g, replace: `$1${MASK}` }, + { re: /\b(\d{8,10}:AA)[A-Za-z0-9_-]{30,}/g, replace: `$1${MASK}` }, + // JWTs — three base64url segments; mask entirely. + { re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,}/g, replace: MASK }, + + // ── 3. Structured secrets ──────────────────────────────────────────────── + // DSN-style credential — a long hex key sitting in the userinfo of a URL + // (`https://@host/…`, e.g. a Sentry DSN). Matched structurally (no + // hostname literal) so it stays host-agnostic and doesn't trip URL linters. + { re: /(https?:\/\/)[a-f0-9]{32,}(@)/gi, replace: `$1${MASK}$2` }, + // Slack incoming webhook — match the distinctive `/services/T…/B…/` + // path (the secret) rather than the hostname literal, so it's host-agnostic + // and doesn't trip URL-anchoring linters. + { re: /(\/services\/)T[A-Z0-9]{6,}\/B[A-Z0-9]{6,}\/[A-Za-z0-9]{20,}/g, replace: `$1${MASK}` }, + // Authorization / Proxy-Authorization headers. + { + re: /((?:proxy-)?authorization["']?\s*[:=]\s*["']?(?:bearer|basic|token|digest)\s+)([A-Za-z0-9._~+/=-]{8,})/gi, + replace: `$1${MASK}`, + }, + // Bare `Bearer `. + { re: /\b(Bearer\s+)([A-Za-z0-9._~+/=-]{12,})/g, replace: `$1${MASK}` }, + // Credentials embedded in a URL: `scheme://user:password@host`. + { re: /([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)([^\s:/@]+)(@)/gi, replace: `$1${MASK}$3` }, + + // ── 4. Generic labeled assignments ─────────────────────────────────────── + { + re: new RegExp( + `((?:${SECRET_KEY_WORDS})(?![A-Za-z0-9])["']?\\s*[:=]\\s*["']?)([^\\s"',;{}()\\[\\]]{1,})`, + "gi", + ), + replace: (match, keep: string, value: string) => + NON_SECRET_VALUE.test(value) ? match : `${keep}${MASK}`, + }, + // CLI-flag form: `--password VALUE` / `-p=VALUE`. + { + re: /(--?(?:password|passwd|pwd|token|secret|api[_-]?key)[\s=]+)(["']?)([^\s"']{1,})/gi, + replace: (match, keep: string, quote: string, value: string) => + NON_SECRET_VALUE.test(value) ? match : `${keep}${quote}${MASK}`, + }, + + // ── 5. High-entropy backstop for bare, unlabeled secrets ───────────────── + // A random-looking token with no known prefix and no labeling key (e.g. an + // AWS secret access key, a raw key echoed in JSON). `/`, `=` and `+` are + // excluded from the candidate charset so file paths, base64 data blobs and + // `key=value` assignments break into short segments — keeping the value + // (a UUID, a decimal, a hash) separate from its label instead of gluing them + // into one high-entropy blob. + { + re: /[A-Za-z0-9_.-]{24,}/g, + replace: (m) => (looksLikeSecret(m) ? MASK : m), + }, +]; + +/** + * Mask tokens, passwords, API keys and other secrets in `text` with stars. + * Returns the input unchanged when it contains no recognized secret. Safe to + * run on a serialized JSON string (introduces only `*` and kept literals). + */ +export function redactSecrets(text: string): string { + if (!text) return text; + let out = text; + for (const rule of RULES) { + out = out.replace(rule.re, rule.replace as string); + } + return out; +} diff --git a/src/hooks/upload-summary.ts b/src/hooks/upload-summary.ts index 6f587c09..1d286676 100644 --- a/src/hooks/upload-summary.ts +++ b/src/hooks/upload-summary.ts @@ -10,6 +10,7 @@ import { randomUUID } from "node:crypto"; import { embeddingSqlLiteral } from "../embeddings/sql.js"; +import { redactSecrets } from "./shared/redact.js"; export type QueryFn = (sql: string) => Promise>>; @@ -124,7 +125,9 @@ export function isFinalizedSummaryText(text: unknown): boolean { * See module docstring for the rationale. */ export async function uploadSummary(query: QueryFn, params: UploadParams): Promise { - const { tableName, vpath, fname, userName, project, agent, text } = params; + const { tableName, vpath, fname, userName, project, agent } = params; + // Mask any secret a summary may have quoted before it's stored/indexed. + const text = redactSecrets(params.text); const ts = params.ts ?? new Date().toISOString(); const desc = extractDescription(text); const sizeBytes = Buffer.byteLength(text); diff --git a/src/hooks/wiki-worker.ts b/src/hooks/wiki-worker.ts index b2174675..2585f45b 100644 --- a/src/hooks/wiki-worker.ts +++ b/src/hooks/wiki-worker.ts @@ -18,6 +18,7 @@ import { deeplakeClientHeader } from "../utils/client-header.js"; const dlog = (msg: string) => _log("wiki-worker", msg); import { finalizeSummary, releaseLock, readState } from "./summary-state.js"; import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "./wiki-offset.js"; +import { redactSecrets } from "./shared/redact.js"; import { uploadSummary } from "./upload-summary.js"; import { embedSummaryWithWarmup } from "../embeddings/embed-summary.js"; import { embeddingsDisabled } from "../embeddings/disable.js"; @@ -284,7 +285,7 @@ async function main(): Promise { if (raw.trim()) { // Stamp the offset ourselves so the persisted summary is authoritative // and never depends on the LLM echoing the bookkeeping line. - const text = stampOffset(raw, jsonlLines); + const text = redactSecrets(stampOffset(raw, jsonlLines)); const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; // Embed the summary so it ranks in the semantic retrieval branch. diff --git a/tests/claude-code/capture-hook.test.ts b/tests/claude-code/capture-hook.test.ts index 822591cc..f0f89653 100644 --- a/tests/claude-code/capture-hook.test.ts +++ b/tests/claude-code/capture-hook.test.ts @@ -210,6 +210,30 @@ describe("capture hook — event-type branches", () => { expect(debugLogMock).toHaveBeenCalledWith(expect.stringMatching(/^tool=Bash session=sid-2$/)); }); + it("tool_call: masks secrets in the tool input/response before insert + embed", async () => { + // Split literal so GitHub push protection doesn't flag this fixture. + const secretToken = "ghp_" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + const secretPw = "s3cr3tP4ssword"; + stdinMock.mockResolvedValue({ + session_id: "sid-secret", + cwd: "/p", + hook_event_name: "PostToolUse", + tool_name: "Bash", + tool_use_id: "tu-2", + tool_input: { command: `git remote set-url origin https://${secretToken}@github.com/o/r` }, + tool_response: { stdout: `PGPASSWORD=${secretPw} psql -h db` }, + }); + await runHook(); + const sql = queryMock.mock.calls[0][0] as string; + // Neither the raw token nor the password reaches the stored row... + expect(sql).not.toContain(secretToken); + expect(sql).not.toContain(secretPw); + // ...but the masked, type-hinted form is present. capture.ts derives the + // embedding from this same redacted `line`, so the secret is never embedded. + expect(sql).toContain("ghp_********"); + expect(sql).toContain("PGPASSWORD=********"); + }); + it("assistant_message without agent_transcript_path", async () => { stdinMock.mockResolvedValue({ session_id: "sid-3", diff --git a/tests/shared/redact.test.ts b/tests/shared/redact.test.ts new file mode 100644 index 00000000..f3220142 --- /dev/null +++ b/tests/shared/redact.test.ts @@ -0,0 +1,246 @@ +import { describe, it, expect } from "vitest"; +import { redactSecrets } from "../../src/hooks/shared/redact.js"; + +const MASK = "********"; + +// Build fixture secrets from split literals at runtime so the *source* never +// contains a scannable vendor token (GitHub push protection / secret scanning +// would otherwise block this very file). The redactor still sees the full +// assembled string. +const j = (...parts: string[]): string => parts.join(""); + +describe("redactSecrets — known token schemes", () => { + const cases: Array<[string, string, string]> = [ + ["GitHub classic PAT", j("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), "ghp_"], + ["GitHub fine-grained PAT", j("github_", "pat_11ABCDE0Y0abcdefghij_KLMNOPqrstuvwxyz0123456789ABCDE"), "github_pat_"], + ["GitHub oauth token", j("gho_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), "gho_"], + ["OpenAI sk key", j("sk-", "ABCDEFGHIJKLMNOPQRSTUVWX"), "sk-"], + ["Anthropic sk-ant key", j("sk-", "ant-api03-ABCDEFGHIJKLMNOPQRSTUV_wx"), "sk-ant-"], + ["Stripe live secret", j("sk_", "live_ABCDEFGHIJKLMNOPQRSTUVWX"), "sk_live_"], + ["Slack bot token", j("xoxb", "-1234567890-abcdefghijABCDEF"), "xoxb-"], + ["Slack app token", j("xapp", "-1-A012345-abcdef"), "xapp-"], + ["Google API key", j("AIza", "SyABCDEFGHIJKLMNOPQRSTUVWXYZ0123456"), "AIza"], + ["HuggingFace token", j("hf_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123"), "hf_"], + ["GitLab PAT", j("glpat", "-ABCDEFGHIJKLMNOPQRST"), "glpat-"], + ]; + for (const [name, secret, keptPrefix] of cases) { + it(`masks ${name} but keeps the scheme prefix`, () => { + const out = redactSecrets(`the value is ${secret} ok`); + expect(out).toContain(`${keptPrefix}${MASK}`); + expect(out).not.toContain(secret); + }); + } + + it("masks an AWS access key id, keeping the AKIA prefix", () => { + const akia = j("AKIA", "IOSFODNN7EXAMPLE"); + const out = redactSecrets(`key ${akia} here`); + expect(out).toContain(`AKIA${MASK}`); + expect(out).not.toContain(akia); + }); + + it("masks a JWT entirely", () => { + const jwt = + j("eyJ", "hbGciOiJIUzI1NiJ9") + "." + j("eyJ", "zdWIiOiIxMjM0NTY3ODkwIn0") + + ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"; + const out = redactSecrets(`token=${jwt}`); + expect(out).not.toContain(jwt); + expect(out).toContain(MASK); + }); +}); + +describe("redactSecrets — headers, URLs, private keys", () => { + it("masks an Authorization: Bearer header", () => { + const out = redactSecrets("Authorization: Bearer abcDEF123456ghiJKL789"); + expect(out).toBe(`Authorization: Bearer ${MASK}`); + }); + + it("masks a bare Bearer token", () => { + const out = redactSecrets("curl -H 'x' Bearer abcDEF123456ghiJKL789xyz"); + expect(out).toContain(`Bearer ${MASK}`); + expect(out).not.toContain("abcDEF123456ghiJKL789xyz"); + }); + + it("masks the password in a connection string, keeping user and host", () => { + const out = redactSecrets("postgres://neohorizon:s3cr3tP4ss@db.internal:5432/app"); + expect(out).toBe(`postgres://neohorizon:${MASK}@db.internal:5432/app`); + }); + + it("strips a PEM private key body, keeping the markers", () => { + const pem = + "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1234\nabcd/efgh+ijkl\n-----END RSA PRIVATE KEY-----"; + const out = redactSecrets(pem); + expect(out).toContain("-----BEGIN RSA PRIVATE KEY-----"); + expect(out).toContain("-----END RSA PRIVATE KEY-----"); + expect(out).toContain(MASK); + expect(out).not.toContain("MIIEpAIBAAKCAQEA1234"); + }); +}); + +describe("redactSecrets — labeled assignments", () => { + const forms: Array<[string, string]> = [ + ["password=hunter2horse", "password="], + ["PASSWORD: hunter2horse", "PASSWORD:"], + ["PGPASSWORD='hunter2horse'", "PGPASSWORD="], + ["export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMIK7MDENGbPxRfiCY", "AWS_SECRET_ACCESS_KEY="], + ["api_key = myS3cretValue123", "api_key ="], + ["client_secret: aBcDeF123456", "client_secret:"], + ]; + for (const [input, keepFragment] of forms) { + it(`masks the value in \`${input}\``, () => { + const out = redactSecrets(input); + expect(out).toContain(keepFragment); + expect(out).toContain(MASK); + // the secret body is gone + expect(out).not.toMatch(/hunter2horse|wJalrXUtnFEMIK7MDENGbPxRfiCY|myS3cretValue123|aBcDeF123456/); + }); + } + + it('masks a JSON "key":"value" pair and stays valid JSON', () => { + const json = JSON.stringify({ tool: "bash", api_key: "supersecretvalue123", note: "hi" }); + const out = redactSecrets(json); + expect(out).not.toContain("supersecretvalue123"); + const parsed = JSON.parse(out); + expect(parsed.api_key).toBe(MASK); + expect(parsed.note).toBe("hi"); + expect(parsed.tool).toBe("bash"); + }); + + it("masks --password CLI flag values (space and = separated)", () => { + expect(redactSecrets("psql --password s3cretPass")).toContain(`--password ${MASK}`); + expect(redactSecrets("mytool --token=abc123def456")).toContain(`--token=${MASK}`); + expect(redactSecrets("psql --password s3cretPass")).not.toContain("s3cretPass"); + }); +}); + +describe("redactSecrets — precision (no over-redaction)", () => { + it("does not touch secret-word look-alikes", () => { + const s = "tokenizer=gpt2 max_tokens=4096 keyboard=mechanical"; + expect(redactSecrets(s)).toBe(s); + }); + + it("leaves boolean/null secret values readable", () => { + for (const s of ["secret: false", "token = null", "password: true", "api_key: none"]) { + expect(redactSecrets(s)).toBe(s); + } + }); + + it("returns non-secret text unchanged", () => { + const s = "SELECT * FROM sessions WHERE org_id = '6a733763' LIMIT 10;"; + expect(redactSecrets(s)).toBe(s); + }); + + it("leaves a CLI flag with a boolean value alone", () => { + expect(redactSecrets("mytool --token=false")).toBe("mytool --token=false"); + }); + + it("does NOT mask a long but low-entropy repeated token", () => { + // Mixed character classes (passes the class gate) but highly repetitive, so + // Shannon entropy stays under the threshold — not a random secret. + const repeated = "Ab1".repeat(10); // 30 chars, entropy ~1.58 bits/char + expect(redactSecrets(`id ${repeated} x`)).toContain(repeated); + }); + + it("handles empty / undefined-ish input", () => { + expect(redactSecrets("")).toBe(""); + // @ts-expect-error — defensive: real callers pass strings + expect(redactSecrets(undefined)).toBe(undefined); + }); +}); + +describe("redactSecrets — extended provider coverage", () => { + const cases: Array<[string, string, string]> = [ + ["npm token", j("npm_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab"), "npm_"], + ["Shopify access token", j("shpat", "_0123456789abcdef0123456789abcdef"), "shpat_"], + ["SendGrid key", j("SG.", "ABCDEFGHIJKLMNOPQRSTUV.") + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab", "SG."], + ["Groq key", j("gsk_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghij"), "gsk_"], + ["xAI key", j("xai-", "A".repeat(70)), "xai-"], + ["Notion secret", j("secret", "_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefg"), "secret_"], + ["Stripe webhook secret", j("whsec_", "ABCDEFGHIJKLMNOPQRSTUVWX"), "whsec_"], + ["Google OAuth access", j("ya29.", "ABCDEFGHIJKLMNOPQRSTUVWX"), "ya29."], + ["Linear api key", j("lin_api_", "ABCDEFGHIJKLMNOPQRSTUVWX"), "lin_api_"], + ["DigitalOcean token", j("dop_v1_", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), "dop_v1_"], + ]; + for (const [name, secret, keptPrefix] of cases) { + it(`masks ${name}`, () => { + const out = redactSecrets(`val=${secret}`); + expect(out).toContain(keptPrefix); + expect(out).not.toContain(secret); + expect(out).toContain(MASK); + }); + } + + it("masks a Telegram bot token", () => { + const tok = j("123456789", ":AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPqrs"); + const out = redactSecrets(`TELEGRAM=${tok}`); + expect(out).not.toContain(tok); + expect(out).toContain(MASK); + }); + + it("masks a Slack incoming-webhook URL path", () => { + const url = j("https://hooks.slack.com/services/", "T00000000/B00000000/abcdefABCDEF0123456789"); + const out = redactSecrets(url); + expect(out).toContain("hooks.slack.com/services/"); + expect(out).not.toContain("abcdefABCDEF0123456789"); + expect(out).toContain(MASK); + }); + + it("masks a Sentry DSN key, keeping the ingest host", () => { + const dsn = j("https://", "0123456789abcdef0123456789abcdef") + "@o123.ingest.sentry.io/456"; + const out = redactSecrets(dsn); + expect(out).toContain("@o123.ingest.sentry.io/456"); + expect(out).not.toContain("0123456789abcdef0123456789abcdef"); + expect(out).toContain(MASK); + }); +}); + +describe("redactSecrets — high-entropy backstop", () => { + it("masks a bare, unlabeled random token with no known prefix", () => { + const secret = "aB3xK9pQ2mN7vR4tW1zY6cF8dG5hJ0"; // 30 chars, mixed classes + const out = redactSecrets(`response body: ${secret} end`); + expect(out).not.toContain(secret); + expect(out).toContain(MASK); + }); + + it("does NOT mask a git SHA (40 hex)", () => { + const sha = "e94b5c716bf6622ac4d41fa513fbff9ee39fb882"; + expect(redactSecrets(`commit ${sha}`)).toContain(sha); + }); + + it("does NOT mask a UUID", () => { + const uuid = "6a733763-1129-4656-aa1f-6f12d4b5c69d"; + expect(redactSecrets(`org_id=${uuid}`)).toContain(uuid); + }); + + it("does NOT mask a long single-case word or a filesystem path", () => { + expect(redactSecrets("abcdefghijklmnopqrstuvwxyzabc")).toBe("abcdefghijklmnopqrstuvwxyzabc"); + const path = "/home/admin/sasun/work/deeplake-api/internal/workspaces"; + expect(redactSecrets(path)).toBe(path); + }); + + it("does NOT mask a long decimal number", () => { + expect(redactSecrets("count=123456789012345678901234567890")).toContain("123456789012345678901234567890"); + }); +}); + +describe("redactSecrets — idempotency & multi-secret", () => { + it("is idempotent", () => { + const s = `GITHUB_TOKEN=${j("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")} and password=hunter2horse`; + const once = redactSecrets(s); + expect(redactSecrets(once)).toBe(once); + }); + + it("masks every secret in a multi-secret blob", () => { + const ghToken = j("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); + const oaKey = j("sk-", "ABCDEFGHIJKLMNOPQRSTUVWX"); + const blob = [ + `export GITHUB_TOKEN=${ghToken}`, + `export OPENAI_API_KEY=${oaKey}`, + "psql postgres://u:p4ssw0rdX@h/db", + ].join("\n"); + const out = redactSecrets(blob); + expect(out).not.toContain(ghToken); + expect(out).not.toContain(oaKey); + expect(out).not.toContain("p4ssw0rdX"); + expect(out.match(/\*{8}/g)?.length).toBeGreaterThanOrEqual(3); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index d73e1742..3f851e72 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -90,6 +90,14 @@ export default defineConfig({ functions: 90, lines: 90, }, + // PR #307 — feat: mask secrets on capture. The redactor is fully + // unit-tested (recall + precision + entropy backstop); lock at 90. + "src/hooks/shared/redact.ts": { + statements: 90, + branches: 90, + functions: 90, + lines: 90, + }, // PR #60 — fix/grep-dual-table-and-normalize. // Raised to 90 to surface the red path in the PR coverage comment // for metrics that sit between 80 and 90 (e.g. grep-core branches