From 47a8046d10fbcd7357b19d6d131b3a188fde5b16 Mon Sep 17 00:00:00 2001 From: lBroth Date: Sat, 30 May 2026 10:31:17 +0200 Subject: [PATCH] feat(threshold): raise default private_date post-filter to 0.85 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GLiNER model is over-eager on bare calendar dates without identifying context — a system-prompt line like "Today's date is 2026-05-21" or a copyright footer like "© 2024 Acme Inc." gets tagged as `private_date` and erodes the token budget on every roundtrip without protecting anyone. Add `DEFAULT_CATEGORY_THRESHOLDS` with `private_date: 0.85` and merge it into the post-filter on a key-by-key basis — user-supplied `categoryThresholds` still win for the labels they spell out. Spans above 0.85 (DOB, birth-date, explicit personal-date PII) keep firing; the long tail of generic dates drops. Co-Authored-By: Claude Opus 4.7 --- src/defaults.ts | 15 +++++++++++++- src/nullpii.ts | 11 +++++++++- test/nullpii.test.ts | 49 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/defaults.ts b/src/defaults.ts index 8aac542..44e6686 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -8,7 +8,7 @@ // Strict TS index-access fallbacks (e.g. `arr[i] ?? 0` under // `noUncheckedIndexedAccess`) are NOT user-facing defaults and stay inline. -import type { BackendName, ModelVariant } from './types/index.js'; +import type { BackendName, ModelVariant, PiiCategory } from './types/index.js'; import type { Recognizer } from './types/recognizer.js'; import { base58CheckValid, @@ -833,6 +833,19 @@ export const DEFAULT_DECODE_THRESHOLD = 0.5; * Override via `NullPiiConfig.threshold` / `categoryThresholds`. */ export const DEFAULT_POST_FILTER_THRESHOLD = 0; +/** Per-category post-filter thresholds layered on top of the global + * default. User-supplied `categoryThresholds` win on a key-by-key + * basis (only the labels the user spells out are overridden). + * + * `private_date` — model is over-eager on calendar dates without + * identifying context (e.g. "Today's date is 2026-05-21" in a system + * prompt or footer). Tightening to 0.85 drops the long tail of + * low-confidence date hits while keeping DOB / birth-date / explicit + * date PII (which the model emits well above 0.9). */ +export const DEFAULT_CATEGORY_THRESHOLDS: Partial> = { + private_date: 0.85, +}; + /** IoU threshold used by `dedupeOverlappingSpans` when reconciling * ML + recognizer spans. Two spans with IoU at or above this are * treated as duplicates; higher-scoring one wins. */ diff --git a/src/nullpii.ts b/src/nullpii.ts index 2ea9a1f..ce999aa 100644 --- a/src/nullpii.ts +++ b/src/nullpii.ts @@ -7,6 +7,7 @@ import { NULLPII_MODEL_DIR, readEnvVar } from './config.js'; import { BOUNDARY_REFINE_TRIM_CHARS, DEFAULT_BOUNDARY_REFINE, + DEFAULT_CATEGORY_THRESHOLDS, DEFAULT_DECODE_THRESHOLD, DEFAULT_DEDUPE_IOU, DEFAULT_POST_FILTER_THRESHOLD, @@ -225,10 +226,18 @@ export class NullPii { DEFAULT_DEDUPE_IOU, { acrossLabels: true }, ) as PiiSpan[]; + // Merge built-in per-category defaults with user overrides — user + // keys win on a key-by-key basis. The built-in `private_date: 0.85` + // drops the long tail of low-confidence calendar-date hits the model + // emits without identifying context. + const mergedCategoryThresholds = { + ...DEFAULT_CATEGORY_THRESHOLDS, + ...(this.config.categoryThresholds ?? {}), + }; const merged = applyThresholds( combined, this.config.threshold ?? DEFAULT_POST_FILTER_THRESHOLD, - this.config.categoryThresholds ?? {}, + mergedCategoryThresholds, ); const refineOn = this.config.boundaryRefine ?? DEFAULT_BOUNDARY_REFINE; const spans = refineOn ? refineSpanBoundaries(escaped, merged) : merged; diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index e91252d..0d04f97 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -198,6 +198,55 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await n.dispose(); }); + it('drops low-confidence private_date spans via built-in category threshold', async () => { + // Default `categoryThresholds.private_date` = 0.85 — a recognizer + // that fires at 0.7 must be dropped; one at 0.9 must survive. This + // protects against the gateway-report case of innocuous calendar + // dates in system prompts being placeholdered. + const n = new NullPii({ + modelDir: '/fake', + backend: 'cpu', + recognizers: [ + { + id: 'test:weak-date', + pattern: /\b2026-05-21\b/g, + label: 'private_date', + confidence: 0.7, + }, + { + id: 'test:strong-date', + pattern: /\b1985-03-12\b/g, + label: 'private_date', + confidence: 0.9, + }, + ], + }); + const out = await n.sanitize('Today 2026-05-21 — DOB 1985-03-12'); + expect(out.spans).toHaveLength(1); + expect(out.spans[0]?.text).toBe('1985-03-12'); + await n.dispose(); + }); + + it('user categoryThresholds override built-in date default per-key', async () => { + // Caller can lower the bar back down for their own use case. + const n = new NullPii({ + modelDir: '/fake', + backend: 'cpu', + categoryThresholds: { private_date: 0.5 }, + recognizers: [ + { + id: 'test:weak-date', + pattern: /\b2026-05-21\b/g, + label: 'private_date', + confidence: 0.7, + }, + ], + }); + const out = await n.sanitize('Today 2026-05-21'); + expect(out.spans).toHaveLength(1); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io');