From dbf6936b8834c2a6a5b05e8a99e10b65a93db68d Mon Sep 17 00:00:00 2001 From: Raphael Mobis Tacla Date: Mon, 23 Mar 2026 23:09:02 -0300 Subject: [PATCH 1/3] refactor lint:links command and remove random cache purge --- bin/validate-links.mts | 107 +++++++++++++---------------------- package-lock.json | 124 ----------------------------------------- package.json | 4 -- 3 files changed, 39 insertions(+), 196 deletions(-) diff --git a/bin/validate-links.mts b/bin/validate-links.mts index 0249805..60171da 100644 --- a/bin/validate-links.mts +++ b/bin/validate-links.mts @@ -3,19 +3,33 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { basename } from 'node:path'; import { exit } from 'node:process'; import type { ValidateFunction } from 'ajv'; -import CLIProgress from 'cli-progress'; import { Glob } from 'glob'; -import runParallelLimit from 'run-parallel-limit'; import type { TaskTier } from '@/types.js'; import ajv from '@/util/ajv.mjs'; +const CACHE_DIR = './.cache/links'; const IMAGE_CONTENT_TYPE_REGEX = /^image\/(png|gif)/; +const HTML_CONTENT_TYPE_REGEX = /^text\/html/; function cacheKey(link: string): string { return hash('sha1', link); } -const progressBar = new CLIProgress.SingleBar({}, CLIProgress.Presets.shades_grey); +async function validateLink(link: string, contentTypeRegex: RegExp): Promise { + if (cachedKeys.has(cacheKey(link))) { + return true; + } + + const res = await fetch(link, { method: 'HEAD' }); + const contentType = res.headers.get('Content-Type'); + + if (res.status !== 200 || !contentType?.match(contentTypeRegex)) { + return false; + } + + await writeFile(`${CACHE_DIR}/${cacheKey(link)}`, ''); + return true; +} // type "casting" is required here for type guards to work const validateTier = ajv.getSchema( @@ -23,29 +37,17 @@ const validateTier = ajv.getSchema( ) as ValidateFunction; // create cache directory if needed -await mkdir('./.cache/links', { recursive: true }); +await mkdir(CACHE_DIR, { recursive: true }); const cachedKeys = new Set(); -const cacheWalker = new Glob('./.cache/links/*', {}); +const cacheWalker = new Glob(`${CACHE_DIR}/*`, {}); for await (const cacheFile of cacheWalker) { cachedKeys.add(basename(cacheFile)); } console.log(`Found ${cachedKeys.size} cached links!`); -// randomly purge 10% of cache so we continuously check part of past links -// this might help us catch any changed links -let purgedKeys = 0; -for (const cachedKey of cachedKeys) { - if (Math.random() < 0.2) { - cachedKeys.delete(cachedKey); - purgedKeys++; - } -} - -console.log(`Randomly purged ${purgedKeys} links from cache!`); - const wikiLinks = new Set(); const imageLinks = new Set(); @@ -63,56 +65,25 @@ for await (const tierFile of tierWalker) { } } -const wikiTasks = wikiLinks - .values() - .filter((link) => !cachedKeys.has(cacheKey(link))) - .map((link) => async (cb: (err: Error | null, results: string | null) => void) => { - const res = await fetch(link, { method: 'HEAD' }); - const contentType = res.headers.get('Content-Type'); - - progressBar.increment(); - if (res.status !== 200 || !contentType?.startsWith('text/html')) { - console.log(res); - return cb(null, link); - } - - await writeFile(`./.cache/links/${cacheKey(link)}`, ''); - return cb(null, null); - }); - -const imageTasks = imageLinks - .values() - .filter((link) => !cachedKeys.has(cacheKey(link))) - .map((link) => async (cb: (err: Error | null, results: string | null) => void) => { - const res = await fetch(link, { method: 'HEAD' }); - const contentType = res.headers.get('Content-Type'); - - progressBar.increment(); - if (res.status !== 200 || !contentType?.match(IMAGE_CONTENT_TYPE_REGEX)) { - console.log(res); - return cb(null, link); - } - - await writeFile(`./.cache/links/${cacheKey(link)}`, ''); - return cb(null, null); - }); - -const allTasks = [...wikiTasks, ...imageTasks]; - -console.log(`Checking ${allTasks.length} links...`); -progressBar.start(allTasks.length, 0); - -runParallelLimit(allTasks, 1, (_, res) => { - progressBar.stop(); - - const invalidLinks = res.filter(Boolean) as string[]; - if (invalidLinks.length > 0) { - console.error(); - console.error(`${invalidLinks.length} invalid links found:`); - for (const link of invalidLinks) { - console.error(`- ${link}`); - } +let hasErrors = false; - exit(1); +console.log(`Checking ${wikiLinks.size} wiki links...`); +for (const link of wikiLinks) { + if (!(await validateLink(link, HTML_CONTENT_TYPE_REGEX))) { + hasErrors = true; + console.error(`- Invalid link: ${link}`); + } +} + +console.log(`Checking ${imageLinks.size} image links...`); +for (const link of imageLinks) { + if (!(await validateLink(link, IMAGE_CONTENT_TYPE_REGEX))) { + hasErrors = true; + console.error(`- Invalid link: ${link}`); } -}); +} + +if (hasErrors) { + console.error('Invalid links found!'); + exit(1); +} diff --git a/package-lock.json b/package-lock.json index 227aca8..e212ab5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,18 +9,14 @@ "@tsconfig/node-ts": "^23.6.1", "@tsconfig/node22": "^22.0.2", "@types/ajv": "^0.0.5", - "@types/cli-progress": "^3.11.6", "@types/node": "^24.0.13", - "@types/run-parallel-limit": "^1.0.3", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", - "cli-progress": "^3.12.0", "fracturedjsonjs": "^4.1.0", "glob": "^11.0.3", "json-schema-to-typescript": "^15.0.4", "lefthook": "^1.12.1", "radash": "^12.1.1", - "run-parallel-limit": "^1.1.0", "tsx": "^4.20.3", "typescript": "^5.8.3", "ultracite": "5.0.32" @@ -1038,16 +1034,6 @@ "@types/deep-eql": "*" } }, - "node_modules/@types/cli-progress": { - "version": "3.11.6", - "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.6.tgz", - "integrity": "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -1086,13 +1072,6 @@ "undici-types": "~7.8.0" } }, - "node_modules/@types/run-parallel-limit": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@types/run-parallel-limit/-/run-parallel-limit-1.0.3.tgz", - "integrity": "sha512-lHTi6vjizim5MPAgkW38k+PbovIBpUr3bo1v4IzCdUCdwnL7M64U5raqPz7kTGhv9Jd7VQ3oxadyNHokAm5QKA==", - "dev": true, - "license": "MIT" - }, "node_modules/@vitest/expect": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", @@ -1323,64 +1302,6 @@ "node": ">= 16" } }, - "node_modules/cli-progress": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", - "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "string-width": "^4.2.3" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/cli-progress/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-progress/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/cli-progress/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-progress/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2151,27 +2072,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/radash": { "version": "12.1.1", "resolved": "https://registry.npmjs.org/radash/-/radash-12.1.1.tgz", @@ -2242,30 +2142,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/run-parallel-limit": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz", - "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index 83c61be..fbd2f4a 100644 --- a/package.json +++ b/package.json @@ -19,18 +19,14 @@ "@tsconfig/node-ts": "^23.6.1", "@tsconfig/node22": "^22.0.2", "@types/ajv": "^0.0.5", - "@types/cli-progress": "^3.11.6", "@types/node": "^24.0.13", - "@types/run-parallel-limit": "^1.0.3", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", - "cli-progress": "^3.12.0", "fracturedjsonjs": "^4.1.0", "glob": "^11.0.3", "json-schema-to-typescript": "^15.0.4", "lefthook": "^1.12.1", "radash": "^12.1.1", - "run-parallel-limit": "^1.1.0", "tsx": "^4.20.3", "typescript": "^5.8.3", "ultracite": "5.0.32" From 8bb9b94a13b4d689608ce5626effb5be8660f97f Mon Sep 17 00:00:00 2001 From: Raphael Mobis Tacla Date: Mon, 23 Mar 2026 23:10:56 -0300 Subject: [PATCH 2/3] fix link validation job name --- .github/workflows/lint-and-check-gen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-check-gen.yml b/.github/workflows/lint-and-check-gen.yml index cf47315..154165a 100644 --- a/.github/workflows/lint-and-check-gen.yml +++ b/.github/workflows/lint-and-check-gen.yml @@ -42,7 +42,7 @@ jobs: restore-keys: | cache- - - name: Lint code + - name: Lint links run: npm run lint:links check-types: From 8943113a3902c586312ae1f4f7a88aa5b5b5925f Mon Sep 17 00:00:00 2001 From: Raphael Mobis Tacla Date: Mon, 23 Mar 2026 23:11:21 -0300 Subject: [PATCH 3/3] add continuous link linting job --- .github/workflows/continuous-link-linting.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/continuous-link-linting.yml diff --git a/.github/workflows/continuous-link-linting.yml b/.github/workflows/continuous-link-linting.yml new file mode 100644 index 0000000..1279482 --- /dev/null +++ b/.github/workflows/continuous-link-linting.yml @@ -0,0 +1,19 @@ +name: Continuous Link Linting + +on: + schedule: + - cron: 0 0 * * * + +jobs: + lint-links: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: ./.github/actions/setup + + # we intentionally do not restore cache here + - name: Lint links + run: npm run lint:links