Skip to content

feat(batch): run heuristic search even when Phase 1 succeeds for force_search_ahead sites #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Alex-302 merged 1 commit into from
May 8, 2026
Merged

feat(batch): run heuristic search even when Phase 1 succeeds for force_search_ahead sites #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 145 additions & 11 deletions __tests__/batch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -897,13 +897,14 @@ describe('8. force_search_ahead scenarios', () => {
let callCount = 0;
jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
if (callCount === 1) {
const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname;
if (host === 'testsite1.com') {
return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true }));
}
// First candidate succeeds
if (url.includes('testsite2.com')) return Promise.resolve(makeSuccessResult('testsite2.com'));
if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com'));
// Second candidate (should not be checked due to early stop)
if (url.includes('testsite3.com')) return Promise.resolve(makeSuccessResult('testsite3.com'));
if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com'));
return Promise.resolve(makeFailResult('Not found'));
});

Expand Down Expand Up @@ -935,19 +936,20 @@ describe('8. force_search_ahead scenarios', () => {

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
if (callCount === 1) {
const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname;
if (host === 'testsite1.com') {
return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true }));
}
// Multiple candidates succeed
if (url.includes('testsite2.com')) {
if (host === 'testsite2.com') {
successfulCandidates.push('testsite2.com');
return Promise.resolve(makeSuccessResult('testsite2.com'));
}
if (url.includes('testsite3.com')) {
if (host === 'testsite3.com') {
successfulCandidates.push('testsite3.com');
return Promise.resolve(makeSuccessResult('testsite3.com'));
}
if (url.includes('testsite4.com')) {
if (host === 'testsite4.com') {
successfulCandidates.push('testsite4.com');
return Promise.resolve(makeSuccessResult('testsite4.com'));
}
Expand Down Expand Up @@ -983,19 +985,20 @@ describe('8. force_search_ahead scenarios', () => {

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
if (callCount === 1) {
const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname;
if (host === 'testsite1.com') {
return Promise.resolve(makeFailResult('DNS failed', { shouldTriggerHeuristic: true }));
}
// Candidate 2: success with correct content
if (url.includes('testsite2.com')) {
if (host === 'testsite2.com') {
return Promise.resolve(makeSuccessResult('testsite2.com', { finalBody: 'Expected Content here' }));
}
// Candidate 3: success but wrong content (probe fails)
if (url.includes('testsite3.com')) {
if (host === 'testsite3.com') {
return Promise.resolve(makeSuccessResult('testsite3.com', { finalBody: 'Wrong Content' }));
}
// Candidate 4: success with correct content
if (url.includes('testsite4.com')) {
if (host === 'testsite4.com') {
return Promise.resolve(makeSuccessResult('testsite4.com', { finalBody: 'Expected Content again' }));
}
return Promise.resolve(makeFailResult('Not found'));
Expand All @@ -1010,6 +1013,137 @@ describe('8. force_search_ahead scenarios', () => {
// Should check multiple candidates
expect(callCount).toBeGreaterThan(2);
});

test('8.4 force_search_ahead: Phase 1 success → still generates candidates and collects additional domains', async () => {
const config = makeConfig({
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 5, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: true,
});
const watchers = makeWatchers({ 'testsite': site });
const logger = makeLogger();
const resolver = new HttpResolver(config);

let callCount = 0;

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname;
// Phase 1: testsite1.com redirects to testsite2.com
if (host === 'testsite1.com') {
return Promise.resolve(makeSuccessResult('testsite2.com'));
}
// Phase 2: Heuristic candidates — multiple succeed
if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com'));
if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com'));
if (host === 'testsite4.com') return Promise.resolve(makeSuccessResult('testsite4.com'));
if (host === 'testsite5.com') return Promise.resolve(makeSuccessResult('testsite5.com'));
return Promise.resolve(makeFailResult('Not found'));
});

const processor = new BatchProcessor(config, watchers, logger, resolver);
const results = await processor.processAll();

// Phase 1 found testsite2.com (redirect from testsite1.com)
expect(results[0].newHost).toBe('testsite2.com');
expect(results[0].hostChanged).toBe(true);
// Heuristic should have run and collected additional working domains
expect(results[0].additionalWorkingDomains).toEqual(
expect.arrayContaining(['testsite3.com', 'testsite4.com', 'testsite5.com'])
);
// Should have made initial call + heuristic candidates
expect(callCount).toBeGreaterThan(3);
});

test('8.5 force_search_ahead: Phase 1 success but host unchanged → still generates candidates', async () => {
const config = makeConfig({
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 3, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: true,
});
const watchers = makeWatchers({ 'testsite': site });
const logger = makeLogger();
const resolver = new HttpResolver(config);

let callCount = 0;

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
const host = new URL(url.startsWith('http') ? url : `https://${url}`).hostname;
// Phase 1: succeeds but host unchanged
if (host === 'testsite1.com') {
return Promise.resolve(makeSuccessResult('testsite1.com'));
}
// Phase 2: Heuristic finds new working domains
if (host === 'testsite2.com') return Promise.resolve(makeSuccessResult('testsite2.com'));
if (host === 'testsite3.com') return Promise.resolve(makeSuccessResult('testsite3.com'));
return Promise.resolve(makeFailResult('Not found'));
});

const processor = new BatchProcessor(config, watchers, logger, resolver);
const results = await processor.processAll();

// Phase 1 returned testsite1.com (no host change)
// Heuristic should still run due to force_search_ahead and collect additional domains
expect(results[0].newHost).toBe('testsite1.com');
expect(results[0].additionalWorkingDomains).toEqual(
expect.arrayContaining(['testsite2.com', 'testsite3.com'])
);
});

test('8.6 force_search_ahead: collects final working domains from all redirects', async () => {
const config = makeConfig({
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 3, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: true,
});
const watchers = makeWatchers({ 'testsite': site });
const logger = makeLogger();
const resolver = new HttpResolver(config);

let callCount = 0;

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
// Phase 1: testsite1.com is alive (no redirect)
if (callCount === 1) {
return Promise.resolve(makeSuccessResult('testsite1.com'));
}
// Phase 2 candidates:
// testsite2.com → alive (responds as itself)
if (url.includes('testsite2.com')) {
return Promise.resolve(makeSuccessResult('testsite2.com'));
}
// testsite3.com → REDIRECTS to different.com (200)
if (url.includes('testsite3.com')) {
return Promise.resolve(makeSuccessResult('different.com'));
}
return Promise.resolve(makeFailResult('Not found'));
});

const processor = new BatchProcessor(config, watchers, logger, resolver);
const results = await processor.processAll();

// Primary domain is testsite1.com (alive, Phase 1)
expect(results[0].newHost).toBe('testsite1.com');

// Collect ALL final working domains, including through redirects
const additional = results[0].additionalWorkingDomains || [];

// testsite2.com → alive → collected
expect(additional).toContain('testsite2.com');
// testsite3.com → redirects to different.com (200) → different.com is collected
expect(additional).toContain('different.com');
});
});

// ============================================================================
Expand Down
26 changes: 21 additions & 5 deletions dist/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15745,7 +15745,7 @@ class BatchProcessor {
try {
const hostname = new URL(url).hostname;
await Promise.race([
external_dns_namespaceObject.promises.resolve(hostname),
external_dns_namespaceObject.promises.lookup(hostname),
new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), timeout))
]);
return true;
Expand All @@ -15755,7 +15755,7 @@ class BatchProcessor {
try {
const hostname = new URL(url).hostname;
await Promise.race([
external_dns_namespaceObject.promises.resolve(hostname),
external_dns_namespaceObject.promises.lookup(hostname),
new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), 2500))
]);
return true;
Expand Down Expand Up @@ -16006,7 +16006,7 @@ class BatchProcessor {
const antibot = r.result.antibotDetected;
const forceHeuristic = r.result.shouldTriggerHeuristic;
const skipHeuristic = Boolean(site.disable_heuristic) || (antibot && this.config.heuristic.skipOnAntibot && !forceHeuristic);
if ((failed || forceHeuristic) && !skipHeuristic) {
if ((failed || forceHeuristic || site.force_search_ahead) && !skipHeuristic) {
const failedUrl = site.initial_domain || site.last_known_mirror;
const candidates = this.generateCandidates(name, i, site, failedUrl);
allTasks.push(...candidates);
Expand Down Expand Up @@ -16051,6 +16051,21 @@ class BatchProcessor {
const heuristicParallel = this.config.processing.heuristicParallel ?? this.config.processing.parallel;
const foundSites = new Set();
const foundDomainsPerSite = new Map();
// Pre-populate foundSites and foundDomainsPerSite with successful Phase 1 results for force_search_ahead sites
// This ensures Phase 2 doesn't overwrite Phase 1 results, only collects additional domains
for (let i = 0; i < siteEntries.length; i++) {
const [name, site] = siteEntries[i];
const r = results[i];
if (r && r.result.success && site.force_search_ahead) {
foundSites.add(i);
foundDomainsPerSite.set(i, [{
domain: r.newHost, // final working domain after redirects
result: r.result,
candidateUrl: r.startedHost,
}]);
this.logger.info(name, `force_search_ahead: Phase 1 working domain ${r.newHost} collected`);
}
}
const activePromises = new Map();
let nextTaskIndex = 0;
const checkCandidate = async (task) => {
Expand Down Expand Up @@ -16124,7 +16139,8 @@ class BatchProcessor {
const chainFormatted = this.resolver.formatRedirectChain(result.redirectChain);
this.logger.info(task.siteName, `Heuristic SUCCESS: ${task.candidateUrl}`);
this.logger.info(task.siteName, `Heuristic redirect chain: ${chainFormatted}`);
// If force_search_ahead, collect multiple domains
// If force_search_ahead, collect the final working domain
// The finalHost is the domain that returned 200 OK, which is what we want
if (task.site.force_search_ahead) {
if (!foundDomainsPerSite.has(task.siteIndex)) {
foundDomainsPerSite.set(task.siteIndex, []);
Expand All @@ -16134,7 +16150,7 @@ class BatchProcessor {
result,
candidateUrl: task.candidateUrl,
});
this.logger.info(task.siteName, `force_search_ahead: collected domain ${newHost}, continuing search...`);
this.logger.info(task.siteName, `force_search_ahead: collected working final domain ${newHost} from ${task.candidateUrl}`);
}
// Mark site as found (first success or non-force_search_ahead)
if (!foundSites.has(task.siteIndex)) {
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "rotating-domains-checker",
"version": "1.1.7",
"version": "1.1.9",
"description": "GitHub Action for checking and updating rotating domains in ad blocking filters",
"main": "dist/index.js",
"type": "module",
Expand Down
24 changes: 21 additions & 3 deletions src/batch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ export class BatchProcessor {
const forceHeuristic = r.result.shouldTriggerHeuristic;
const skipHeuristic = Boolean(site.disable_heuristic) || (antibot && this.config.heuristic.skipOnAntibot && !forceHeuristic);

if ((failed || forceHeuristic) && !skipHeuristic) {
if ((failed || forceHeuristic || site.force_search_ahead) && !skipHeuristic) {
const failedUrl = site.initial_domain || site.last_known_mirror;
const candidates = this.generateCandidates(name, i, site, failedUrl);
allTasks.push(...candidates);
Expand Down Expand Up @@ -485,6 +485,23 @@ export class BatchProcessor {
const heuristicParallel = this.config.processing.heuristicParallel ?? this.config.processing.parallel;
const foundSites = new Set<number>();
const foundDomainsPerSite = new Map<number, Array<{ domain: string; result: RedirectResult; candidateUrl: string }>>();

// Pre-populate foundSites and foundDomainsPerSite with successful Phase 1 results for force_search_ahead sites
// This ensures Phase 2 doesn't overwrite Phase 1 results, only collects additional domains
for (let i = 0; i < siteEntries.length; i++) {
const [name, site] = siteEntries[i];
const r = results[i];
if (r && r.result.success && site.force_search_ahead) {
foundSites.add(i);
foundDomainsPerSite.set(i, [{
domain: r.newHost, // final working domain after redirects
result: r.result,
candidateUrl: r.startedHost,
}]);
this.logger.info(name, `force_search_ahead: Phase 1 working domain ${r.newHost} collected`);
}
}

const activePromises = new Map<number, Promise<{ taskIndex: number; task: HeuristicTask & { dnsOk: boolean }; result: RedirectResult }>>();
let nextTaskIndex = 0;

Expand Down Expand Up @@ -562,7 +579,8 @@ export class BatchProcessor {
this.logger.info(task.siteName, `Heuristic SUCCESS: ${task.candidateUrl}`);
this.logger.info(task.siteName, `Heuristic redirect chain: ${chainFormatted}`);

// If force_search_ahead, collect multiple domains
// If force_search_ahead, collect the final working domain
// The finalHost is the domain that returned 200 OK, which is what we want
if (task.site.force_search_ahead) {
if (!foundDomainsPerSite.has(task.siteIndex)) {
foundDomainsPerSite.set(task.siteIndex, []);
Expand All @@ -572,7 +590,7 @@ export class BatchProcessor {
result,
candidateUrl: task.candidateUrl,
});
this.logger.info(task.siteName, `force_search_ahead: collected domain ${newHost}, continuing search...`);
this.logger.info(task.siteName, `force_search_ahead: collected working final domain ${newHost} from ${task.candidateUrl}`);
}

// Mark site as found (first success or non-force_search_ahead)
Expand Down