From 8a719e3c1291fad22a5ce8f997e9055c319517bc Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Fri, 31 Jul 2026 15:02:41 -0400 Subject: [PATCH 1/3] Preserve DOCX manual line breaks --- sdk/typescript/src/knowledge-base.ts | 1 + sdk/typescript/tests-ts/knowledge-base.test.ts | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts index ad82a3c5..6aaaeeea 100644 --- a/sdk/typescript/src/knowledge-base.ts +++ b/sdk/typescript/src/knowledge-base.ts @@ -193,6 +193,7 @@ function extractDocx(path: string, bytes: Uint8Array): string { return decodeXml( xml .replace(/<\/(?:\w+:)?p\s*>/gu, "\n") + .replace(/<(?:\w+:)?br\b[^>]*\/>/gu, "\n") .replace(/<(?:\w+:)?tab\b[^>]*\/>/gu, "\t") .replace(/<[^>]+>/gu, ""), ); diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts index 79ca83c4..33b69fdb 100644 --- a/sdk/typescript/tests-ts/knowledge-base.test.ts +++ b/sdk/typescript/tests-ts/knowledge-base.test.ts @@ -41,10 +41,10 @@ async function extractedDocuments(path: string): Promise { ); } -function docx(text: string): Uint8Array { +function docx(text: string, secondLine?: string): Uint8Array { return zipSync({ "word/document.xml": strToU8( - `${text}`, + `${text}${secondLine === undefined ? "" : `${secondLine}`}`, ), }); } @@ -111,14 +111,17 @@ describe("scan knowledge bases", () => { join(root, "architecture.pdf"), pdf("Payment service boundary"), ); - await writeFile(join(root, "threat-model.docx"), docx("SSRF & IDOR")); + await writeFile( + join(root, "threat-model.docx"), + docx("SSRF & IDOR", "Review authentication"), + ); const knowledgeBase = await prepareKnowledgeBase([root]); temporaryDirectories.push(knowledgeBase.path); const documents = await extractedDocuments(knowledgeBase.path); expect(documents).toContain("Payment service boundary"); - expect(documents).toContain("SSRF & IDOR\n"); + expect(documents).toContain("SSRF & IDOR\nReview authentication\n"); }); test("cleans up documents and rediscovers directory contents on later runs", async () => { From c7d6b92a4ec1fd8520d1551dd53da35b3c85f59a Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Sat, 1 Aug 2026 10:27:00 -0400 Subject: [PATCH 2/3] Handle paired DOCX break elements --- sdk/typescript/src/knowledge-base.ts | 2 +- sdk/typescript/tests-ts/knowledge-base.test.ts | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts index 6aaaeeea..e150d937 100644 --- a/sdk/typescript/src/knowledge-base.ts +++ b/sdk/typescript/src/knowledge-base.ts @@ -193,7 +193,7 @@ function extractDocx(path: string, bytes: Uint8Array): string { return decodeXml( xml .replace(/<\/(?:\w+:)?p\s*>/gu, "\n") - .replace(/<(?:\w+:)?br\b[^>]*\/>/gu, "\n") + .replace(/<(?:\w+:)?br\b[^>]*>/gu, "\n") .replace(/<(?:\w+:)?tab\b[^>]*\/>/gu, "\t") .replace(/<[^>]+>/gu, ""), ); diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts index 33b69fdb..8404c5c9 100644 --- a/sdk/typescript/tests-ts/knowledge-base.test.ts +++ b/sdk/typescript/tests-ts/knowledge-base.test.ts @@ -41,10 +41,14 @@ async function extractedDocuments(path: string): Promise { ); } -function docx(text: string, secondLine?: string): Uint8Array { +function docx( + text: string, + secondLine?: string, + breakElement = "", +): Uint8Array { return zipSync({ "word/document.xml": strToU8( - `${text}${secondLine === undefined ? "" : `${secondLine}`}`, + `${text}${secondLine === undefined ? "" : `${breakElement}${secondLine}`}`, ), }); } @@ -115,6 +119,10 @@ describe("scan knowledge bases", () => { join(root, "threat-model.docx"), docx("SSRF & IDOR", "Review authentication"), ); + await writeFile( + join(root, "paired-break.docx"), + docx("Authorization", "Review permissions", ""), + ); const knowledgeBase = await prepareKnowledgeBase([root]); temporaryDirectories.push(knowledgeBase.path); @@ -122,6 +130,7 @@ describe("scan knowledge bases", () => { expect(documents).toContain("Payment service boundary"); expect(documents).toContain("SSRF & IDOR\nReview authentication\n"); + expect(documents).toContain("Authorization\nReview permissions\n"); }); test("cleans up documents and rediscovers directory contents on later runs", async () => { From 5920ad8cd30bae2d2d3570507d79babf5694ae56 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Thu, 6 Aug 2026 18:09:33 -0400 Subject: [PATCH 3/3] Preserve DOCX carriage-return breaks --- sdk/typescript/src/knowledge-base.ts | 2 +- sdk/typescript/tests-ts/knowledge-base.test.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts index e150d937..689d8655 100644 --- a/sdk/typescript/src/knowledge-base.ts +++ b/sdk/typescript/src/knowledge-base.ts @@ -193,7 +193,7 @@ function extractDocx(path: string, bytes: Uint8Array): string { return decodeXml( xml .replace(/<\/(?:\w+:)?p\s*>/gu, "\n") - .replace(/<(?:\w+:)?br\b[^>]*>/gu, "\n") + .replace(/<(?:\w+:)?(?:br|cr)\b[^>]*>/gu, "\n") .replace(/<(?:\w+:)?tab\b[^>]*\/>/gu, "\t") .replace(/<[^>]+>/gu, ""), ); diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts index 8404c5c9..63a9bd16 100644 --- a/sdk/typescript/tests-ts/knowledge-base.test.ts +++ b/sdk/typescript/tests-ts/knowledge-base.test.ts @@ -123,6 +123,10 @@ describe("scan knowledge bases", () => { join(root, "paired-break.docx"), docx("Authorization", "Review permissions", ""), ); + await writeFile( + join(root, "carriage-return.docx"), + docx("Authentication", "Review sessions", ""), + ); const knowledgeBase = await prepareKnowledgeBase([root]); temporaryDirectories.push(knowledgeBase.path); @@ -131,6 +135,7 @@ describe("scan knowledge bases", () => { expect(documents).toContain("Payment service boundary"); expect(documents).toContain("SSRF & IDOR\nReview authentication\n"); expect(documents).toContain("Authorization\nReview permissions\n"); + expect(documents).toContain("Authentication\nReview sessions\n"); }); test("cleans up documents and rediscovers directory contents on later runs", async () => {