From e788b39bbfeb183276a7713d218b51b9136e5193 Mon Sep 17 00:00:00 2001 From: Alex-302 Date: Fri, 8 May 2026 15:55:13 +0300 Subject: [PATCH] feat(batch): run heuristic search even when Phase 1 succeeds for force_search_ahead sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `force_search_ahead: true`, heuristic candidates are now generated and checked regardless of Phase 1 outcome (dead or alive last_known_mirror). - Phase 1 success no longer skips Phase 2 for force_search_ahead sites - Pre-populate foundSites/foundDomainsPerSite from Phase 1 result so Phase 2 doesn't overwrite it, only appends additional domains - Collect finalHost (after redirects) as the working domain — e.g. turkifsaclub65.sbs → turkifsaclub.org (200) records turkifsaclub.org - All collected domains go into additionalWorkingDomains and are written to filter rules via replacer Tests: updated 8.4–8.5 with arrayContaining assertions and hostname-based mock detection; added 8.6 verifying redirect target collection --- __tests__/batch.test.ts | 156 +++++++++++++++++++++++++++++++++++++--- dist/index.js | 26 +++++-- package-lock.json | 4 +- package.json | 2 +- src/batch.ts | 24 ++++++- 5 files changed, 190 insertions(+), 22 deletions(-) diff --git a/__tests__/batch.test.ts b/__tests__/batch.test.ts index 325647e..4b3a8ba 100644 --- a/__tests__/batch.test.ts +++ b/__tests__/batch.test.ts @@ -897,13 +897,14 @@ describe('8. force_search_ahead scenarios', () => { let callCount = 0; jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { callCount++; - if (callCount === 1) { + const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname; + if (host === 'testsite1.com') { return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true })); } // First candidate succeeds - if (url.includes('testsite2.com')) return Promise.resolve(makeSuccessResult('testsite2.com')); + if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com')); // Second candidate (should not be checked due to early stop) - if (url.includes('testsite3.com')) return Promise.resolve(makeSuccessResult('testsite3.com')); + if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com')); return Promise.resolve(makeFailResult('Not found')); }); @@ -935,19 +936,20 @@ describe('8. force_search_ahead scenarios', () => { jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { callCount++; - if (callCount === 1) { + const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname; + if (host === 'testsite1.com') { return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true })); } // Multiple candidates succeed - if (url.includes('testsite2.com')) { + if (host === 'testsite2.com') { successfulCandidates.push('testsite2.com'); return Promise.resolve(makeSuccessResult('testsite2.com')); } - if (url.includes('testsite3.com')) { + if (host === 'testsite3.com') { successfulCandidates.push('testsite3.com'); return Promise.resolve(makeSuccessResult('testsite3.com')); } - if (url.includes('testsite4.com')) { + if (host === 'testsite4.com') { successfulCandidates.push('testsite4.com'); return Promise.resolve(makeSuccessResult('testsite4.com')); } @@ -983,19 +985,20 @@ describe('8. force_search_ahead scenarios', () => { jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { callCount++; - if (callCount === 1) { + const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname; + if (host === 'testsite1.com') { return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true })); } // Candidate 2: success with correct content - if (url.includes('testsite2.com')) { + if (host === 'testsite2.com') { return Promise.resolve(makeSuccessResult('testsite2.com', { finalBody: 'Expected Content here' })); } // Candidate 3: success but wrong content (probe fails) - if (url.includes('testsite3.com')) { + if (host === 'testsite3.com') { return Promise.resolve(makeSuccessResult('testsite3.com', { finalBody: 'Wrong Content' })); } // Candidate 4: success with correct content - if (url.includes('testsite4.com')) { + if (host === 'testsite4.com') { return Promise.resolve(makeSuccessResult('testsite4.com', { finalBody: 'Expected Content again' })); } return Promise.resolve(makeFailResult('Not found')); @@ -1010,6 +1013,137 @@ describe('8. force_search_ahead scenarios', () => { // Should check multiple candidates expect(callCount).toBeGreaterThan(2); }); + + test('8.4 force_search_ahead: Phase 1 success → still generates candidates and collects additional domains', async () => { + const config = makeConfig({ + dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false }, + heuristic: { enabled: true, maxAttempts: 5, skipOnAntibot: true, forceHeuristicOnCodes: [] }, + }); + const site = makeSite({ + last_known_mirror: 'testsite1.com', + force_search_ahead: true, + }); + const watchers = makeWatchers({ 'testsite': site }); + const logger = makeLogger(); + const resolver = new HttpResolver(config); + + let callCount = 0; + + jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { + callCount++; + const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname; + // Phase 1: testsite1.com redirects to testsite2.com + if (host === 'testsite1.com') { + return Promise.resolve(makeSuccessResult('testsite2.com')); + } + // Phase 2: Heuristic candidates — multiple succeed + if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com')); + if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com')); + if (host === 'testsite4.com') return Promise.resolve(makeSuccessResult('testsite4.com')); + if (host === 'testsite5.com') return Promise.resolve(makeSuccessResult('testsite5.com')); + return Promise.resolve(makeFailResult('Not found')); + }); + + const processor = new BatchProcessor(config, watchers, logger, resolver); + const results = await processor.processAll(); + + // Phase 1 found testsite2.com (redirect from testsite1.com) + expect(results[0].newHost).toBe('testsite2.com'); + expect(results[0].hostChanged).toBe(true); + // Heuristic should have run and collected additional working domains + expect(results[0].additionalWorkingDomains).toEqual( + expect.arrayContaining(['testsite3.com', 'testsite4.com', 'testsite5.com']) + ); + // Should have made initial call + heuristic candidates + expect(callCount).toBeGreaterThan(3); + }); + + test('8.5 force_search_ahead: Phase 1 success but host unchanged → still generates candidates', async () => { + const config = makeConfig({ + dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false }, + heuristic: { enabled: true, maxAttempts: 3, skipOnAntibot: true, forceHeuristicOnCodes: [] }, + }); + const site = makeSite({ + last_known_mirror: 'testsite1.com', + force_search_ahead: true, + }); + const watchers = makeWatchers({ 'testsite': site }); + const logger = makeLogger(); + const resolver = new HttpResolver(config); + + let callCount = 0; + + jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { + callCount++; + const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname; + // Phase 1: succeeds but host unchanged + if (host === 'testsite1.com') { + return Promise.resolve(makeSuccessResult('testsite1.com')); + } + // Phase 2: Heuristic finds new working domains + if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com')); + if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com')); + return Promise.resolve(makeFailResult('Not found')); + }); + + const processor = new BatchProcessor(config, watchers, logger, resolver); + const results = await processor.processAll(); + + // Phase 1 returned testsite1.com (no host change) + // Heuristic should still run due to force_search_ahead and collect additional domains + expect(results[0].newHost).toBe('testsite1.com'); + expect(results[0].additionalWorkingDomains).toEqual( + expect.arrayContaining(['testsite2.com', 'testsite3.com']) + ); + }); + + test('8.6 force_search_ahead: collects final working domains from all redirects', async () => { + const config = makeConfig({ + dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false }, + heuristic: { enabled: true, maxAttempts: 3, skipOnAntibot: true, forceHeuristicOnCodes: [] }, + }); + const site = makeSite({ + last_known_mirror: 'testsite1.com', + force_search_ahead: true, + }); + const watchers = makeWatchers({ 'testsite': site }); + const logger = makeLogger(); + const resolver = new HttpResolver(config); + + let callCount = 0; + + jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => { + callCount++; + // Phase 1: testsite1.com is alive (no redirect) + if (callCount === 1) { + return Promise.resolve(makeSuccessResult('testsite1.com')); + } + // Phase 2 candidates: + // testsite2.com → alive (responds as itself) + if (url.includes('testsite2.com')) { + return Promise.resolve(makeSuccessResult('testsite2.com')); + } + // testsite3.com → REDIRECTS to different.com (200) + if (url.includes('testsite3.com')) { + return Promise.resolve(makeSuccessResult('different.com')); + } + return Promise.resolve(makeFailResult('Not found')); + }); + + const processor = new BatchProcessor(config, watchers, logger, resolver); + const results = await processor.processAll(); + + // Primary domain is testsite1.com (alive, Phase 1) + expect(results[0].newHost).toBe('testsite1.com'); + + // Collect ALL final working domains, including through redirects + const additional = results[0].additionalWorkingDomains || []; + + // testsite2.com → alive → collected + expect(additional).toContain('testsite2.com'); + // testsite3.com → redirects to different.com (200) → different.com is collected + expect(additional).toContain('different.com'); + }); }); // ============================================================================ diff --git a/dist/index.js b/dist/index.js index 855a11f..ff05bfe 100644 --- a/dist/index.js +++ b/dist/index.js @@ -15745,7 +15745,7 @@ class BatchProcessor { try { const hostname = new URL(url).hostname; await Promise.race([ - external_dns_namespaceObject.promises.resolve(hostname), + external_dns_namespaceObject.promises.lookup(hostname), new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), timeout)) ]); return true; @@ -15755,7 +15755,7 @@ class BatchProcessor { try { const hostname = new URL(url).hostname; await Promise.race([ - external_dns_namespaceObject.promises.resolve(hostname), + external_dns_namespaceObject.promises.lookup(hostname), new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), 2500)) ]); return true; @@ -16006,7 +16006,7 @@ class BatchProcessor { const antibot = r.result.antibotDetected; const forceHeuristic = r.result.shouldTriggerHeuristic; const skipHeuristic = Boolean(site.disable_heuristic) || (antibot && this.config.heuristic.skipOnAntibot && !forceHeuristic); - if ((failed || forceHeuristic) && !skipHeuristic) { + if ((failed || forceHeuristic || site.force_search_ahead) && !skipHeuristic) { const failedUrl = site.initial_domain || site.last_known_mirror; const candidates = this.generateCandidates(name, i, site, failedUrl); allTasks.push(...candidates); @@ -16051,6 +16051,21 @@ class BatchProcessor { const heuristicParallel = this.config.processing.heuristicParallel ?? this.config.processing.parallel; const foundSites = new Set(); const foundDomainsPerSite = new Map(); + // Pre-populate foundSites and foundDomainsPerSite with successful Phase 1 results for force_search_ahead sites + // This ensures Phase 2 doesn't overwrite Phase 1 results, only collects additional domains + for (let i = 0; i < siteEntries.length; i++) { + const [name, site] = siteEntries[i]; + const r = results[i]; + if (r && r.result.success && site.force_search_ahead) { + foundSites.add(i); + foundDomainsPerSite.set(i, [{ + domain: r.newHost, // final working domain after redirects + result: r.result, + candidateUrl: r.startedHost, + }]); + this.logger.info(name, `force_search_ahead: Phase 1 working domain ${r.newHost} collected`); + } + } const activePromises = new Map(); let nextTaskIndex = 0; const checkCandidate = async (task) => { @@ -16124,7 +16139,8 @@ class BatchProcessor { const chainFormatted = this.resolver.formatRedirectChain(result.redirectChain); this.logger.info(task.siteName, `Heuristic SUCCESS: ${task.candidateUrl}`); this.logger.info(task.siteName, `Heuristic redirect chain: ${chainFormatted}`); - // If force_search_ahead, collect multiple domains + // If force_search_ahead, collect the final working domain + // The finalHost is the domain that returned 200 OK, which is what we want if (task.site.force_search_ahead) { if (!foundDomainsPerSite.has(task.siteIndex)) { foundDomainsPerSite.set(task.siteIndex, []); @@ -16134,7 +16150,7 @@ class BatchProcessor { result, candidateUrl: task.candidateUrl, }); - this.logger.info(task.siteName, `force_search_ahead: collected domain ${newHost}, continuing search...`); + this.logger.info(task.siteName, `force_search_ahead: collected working final domain ${newHost} from ${task.candidateUrl}`); } // Mark site as found (first success or non-force_search_ahead) if (!foundSites.has(task.siteIndex)) { diff --git a/package-lock.json b/package-lock.json index d7a8762..4b0a1b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rotating-domains-checker", - "version": "1.1.7", + "version": "1.1.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rotating-domains-checker", - "version": "1.1.7", + "version": "1.1.9", "license": "MIT", "dependencies": { "table": "^6.8.1", diff --git a/package.json b/package.json index 95ae6c8..c46176a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rotating-domains-checker", - "version": "1.1.7", + "version": "1.1.9", "description": "GitHub Action for checking and updating rotating domains in ad blocking filters", "main": "dist/index.js", "type": "module", diff --git a/src/batch.ts b/src/batch.ts index 682fa68..a6e5d53 100644 --- a/src/batch.ts +++ b/src/batch.ts @@ -432,7 +432,7 @@ export class BatchProcessor { const forceHeuristic = r.result.shouldTriggerHeuristic; const skipHeuristic = Boolean(site.disable_heuristic) || (antibot && this.config.heuristic.skipOnAntibot && !forceHeuristic); - if ((failed || forceHeuristic) && !skipHeuristic) { + if ((failed || forceHeuristic || site.force_search_ahead) && !skipHeuristic) { const failedUrl = site.initial_domain || site.last_known_mirror; const candidates = this.generateCandidates(name, i, site, failedUrl); allTasks.push(...candidates); @@ -485,6 +485,23 @@ export class BatchProcessor { const heuristicParallel = this.config.processing.heuristicParallel ?? this.config.processing.parallel; const foundSites = new Set(); const foundDomainsPerSite = new Map>(); + + // Pre-populate foundSites and foundDomainsPerSite with successful Phase 1 results for force_search_ahead sites + // This ensures Phase 2 doesn't overwrite Phase 1 results, only collects additional domains + for (let i = 0; i < siteEntries.length; i++) { + const [name, site] = siteEntries[i]; + const r = results[i]; + if (r && r.result.success && site.force_search_ahead) { + foundSites.add(i); + foundDomainsPerSite.set(i, [{ + domain: r.newHost, // final working domain after redirects + result: r.result, + candidateUrl: r.startedHost, + }]); + this.logger.info(name, `force_search_ahead: Phase 1 working domain ${r.newHost} collected`); + } + } + const activePromises = new Map>(); let nextTaskIndex = 0; @@ -562,7 +579,8 @@ export class BatchProcessor { this.logger.info(task.siteName, `Heuristic SUCCESS: ${task.candidateUrl}`); this.logger.info(task.siteName, `Heuristic redirect chain: ${chainFormatted}`); - // If force_search_ahead, collect multiple domains + // If force_search_ahead, collect the final working domain + // The finalHost is the domain that returned 200 OK, which is what we want if (task.site.force_search_ahead) { if (!foundDomainsPerSite.has(task.siteIndex)) { foundDomainsPerSite.set(task.siteIndex, []); @@ -572,7 +590,7 @@ export class BatchProcessor { result, candidateUrl: task.candidateUrl, }); - this.logger.info(task.siteName, `force_search_ahead: collected domain ${newHost}, continuing search...`); + this.logger.info(task.siteName, `force_search_ahead: collected working final domain ${newHost} from ${task.candidateUrl}`); } // Mark site as found (first success or non-force_search_ahead)