lint:links improvements

.github/workflows/continuous-link-linting.yml

@@ -0,0 +1,19 @@
+name: Continuous Link Linting
+
+on:
+  schedule:
+    - cron: 0 0 * * *
+
+jobs:
+  lint-links:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: ./.github/actions/setup
+
+      # we intentionally do not restore cache here
+      - name: Lint links
+        run: npm run lint:links

.github/workflows/lint-and-check-gen.yml

@@ -42,7 +42,7 @@ jobs:
           restore-keys: |
             cache-
 
-      - name: Lint code
+      - name: Lint links
         run: npm run lint:links
 
   check-types:

bin/validate-links.mts

@@ -3,49 +3,51 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { basename } from 'node:path';
 import { exit } from 'node:process';
 import type { ValidateFunction } from 'ajv';
-import CLIProgress from 'cli-progress';
 import { Glob } from 'glob';
-import runParallelLimit from 'run-parallel-limit';
 import type { TaskTier } from '@/types.js';
 import ajv from '@/util/ajv.mjs';
 
+const CACHE_DIR = './.cache/links';
 const IMAGE_CONTENT_TYPE_REGEX = /^image\/(png|gif)/;
+const HTML_CONTENT_TYPE_REGEX = /^text\/html/;
 
 function cacheKey(link: string): string {
 	return hash('sha1', link);
 }
 
-const progressBar = new CLIProgress.SingleBar({}, CLIProgress.Presets.shades_grey);
+async function validateLink(link: string, contentTypeRegex: RegExp): Promise<boolean> {
+	if (cachedKeys.has(cacheKey(link))) {
+		return true;
+	}
+
+	const res = await fetch(link, { method: 'HEAD' });
+	const contentType = res.headers.get('Content-Type');
+
+	if (res.status !== 200 || !contentType?.match(contentTypeRegex)) {
+		return false;
+	}
+
+	await writeFile(`${CACHE_DIR}/${cacheKey(link)}`, '');
+	return true;
+}
 
 // type "casting" is required here for type guards to work
 const validateTier = ajv.getSchema(
 	'http://osrs-taskman.com/task-tier.schema.json'
 ) as ValidateFunction<TaskTier>;
 
 // create cache directory if needed
-await mkdir('./.cache/links', { recursive: true });
+await mkdir(CACHE_DIR, { recursive: true });
 
 const cachedKeys = new Set<string>();
 
-const cacheWalker = new Glob('./.cache/links/*', {});
+const cacheWalker = new Glob(`${CACHE_DIR}/*`, {});
 for await (const cacheFile of cacheWalker) {
 	cachedKeys.add(basename(cacheFile));
 }
 
 console.log(`Found ${cachedKeys.size} cached links!`);
 
-// randomly purge 10% of cache so we continuously check part of past links
-// this might help us catch any changed links
-let purgedKeys = 0;
-for (const cachedKey of cachedKeys) {
-	if (Math.random() < 0.2) {
-		cachedKeys.delete(cachedKey);
-		purgedKeys++;
-	}
-}
-
-console.log(`Randomly purged ${purgedKeys} links from cache!`);
-
 const wikiLinks = new Set<string>();
 const imageLinks = new Set<string>();
 
@@ -63,56 +65,25 @@ for await (const tierFile of tierWalker) {
 	}
 }
 
-const wikiTasks = wikiLinks
-	.values()
-	.filter((link) => !cachedKeys.has(cacheKey(link)))
-	.map((link) => async (cb: (err: Error | null, results: string | null) => void) => {
-		const res = await fetch(link, { method: 'HEAD' });
-		const contentType = res.headers.get('Content-Type');
-
-		progressBar.increment();
-		if (res.status !== 200 || !contentType?.startsWith('text/html')) {
-			console.log(res);
-			return cb(null, link);
-		}
-
-		await writeFile(`./.cache/links/${cacheKey(link)}`, '');
-		return cb(null, null);
-	});
-
-const imageTasks = imageLinks
-	.values()
-	.filter((link) => !cachedKeys.has(cacheKey(link)))
-	.map((link) => async (cb: (err: Error | null, results: string | null) => void) => {
-		const res = await fetch(link, { method: 'HEAD' });
-		const contentType = res.headers.get('Content-Type');
-
-		progressBar.increment();
-		if (res.status !== 200 || !contentType?.match(IMAGE_CONTENT_TYPE_REGEX)) {
-			console.log(res);
-			return cb(null, link);
-		}
-
-		await writeFile(`./.cache/links/${cacheKey(link)}`, '');
-		return cb(null, null);
-	});
-
-const allTasks = [...wikiTasks, ...imageTasks];
-
-console.log(`Checking ${allTasks.length} links...`);
-progressBar.start(allTasks.length, 0);
-
-runParallelLimit(allTasks, 1, (_, res) => {
-	progressBar.stop();
-
-	const invalidLinks = res.filter(Boolean) as string[];
-	if (invalidLinks.length > 0) {
-		console.error();
-		console.error(`${invalidLinks.length} invalid links found:`);
-		for (const link of invalidLinks) {
-			console.error(`- ${link}`);
-		}
+let hasErrors = false;
 
-		exit(1);
+console.log(`Checking ${wikiLinks.size} wiki links...`);
+for (const link of wikiLinks) {
+	if (!(await validateLink(link, HTML_CONTENT_TYPE_REGEX))) {
+		hasErrors = true;
+		console.error(`- Invalid link: ${link}`);
+	}
+}
+
+console.log(`Checking ${imageLinks.size} image links...`);
+for (const link of imageLinks) {
+	if (!(await validateLink(link, IMAGE_CONTENT_TYPE_REGEX))) {
+		hasErrors = true;
+		console.error(`- Invalid link: ${link}`);
 	}
-});
+}
+
+if (hasErrors) {
+	console.error('Invalid links found!');
+	exit(1);
+}

package.json

@@ -19,18 +19,14 @@
 		"@tsconfig/node-ts": "^23.6.1",
 		"@tsconfig/node22": "^22.0.2",
 		"@types/ajv": "^0.0.5",
-		"@types/cli-progress": "^3.11.6",
 		"@types/node": "^24.0.13",
-		"@types/run-parallel-limit": "^1.0.3",
 		"ajv": "^8.17.1",
 		"ajv-formats": "^3.0.1",
-		"cli-progress": "^3.12.0",
 		"fracturedjsonjs": "^4.1.0",
 		"glob": "^11.0.3",
 		"json-schema-to-typescript": "^15.0.4",
 		"lefthook": "^1.12.1",
 		"radash": "^12.1.1",
-		"run-parallel-limit": "^1.1.0",
 		"tsx": "^4.20.3",
 		"typescript": "^5.8.3",
 		"ultracite": "5.0.32"