Skip to content

fix: replace dns.resolve() with dns.lookup() for reliable DNS resolution #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Alex-302 merged 1 commit into from
May 8, 2026
Merged

fix: replace dns.resolve() with dns.lookup() for reliable DNS resolution #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 8 additions & 19 deletions __tests__/batch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ function makeFailResult(error: string, overrides: Partial<RedirectResult> = {}):
// ============================================================================
// 3. generateCandidates (tested via processAll with heuristic)
// ============================================================================

describe('3. Heuristic candidate generation', () => {
test('3.1 domain[N].tld: example001.com → generates example002..006', async () => {
const config = makeConfig({
Expand Down Expand Up @@ -319,7 +318,6 @@ describe('3. Heuristic candidate generation', () => {
// ============================================================================
// 4. shouldUpdate logic
// ============================================================================

describe('4. shouldUpdate logic', () => {
test('4.1 hostChanged: true → shouldUpdate: true', async () => {
const config = makeConfig({ dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false } });
Expand Down Expand Up @@ -408,7 +406,6 @@ describe('4. shouldUpdate logic', () => {
// ============================================================================
// 4.7 calculateDaysSince (tested indirectly via processSite optimization)
// ============================================================================

describe('4.7 calculateDaysSince (via recent last_seen optimization)', () => {
test('last_seen recent (< 2 days) → tries last_known_mirror first', async () => {
const config = makeConfig({ dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false } });
Expand Down Expand Up @@ -477,7 +474,6 @@ describe('4.7 calculateDaysSince (via recent last_seen optimization)', () => {
// ============================================================================
// 5. Heuristic triggering conditions
// ============================================================================

describe('5. Heuristic triggering conditions', () => {
test('5.1 antibot + skipOnAntibot: true + accept_antibot: false → foundSites.add, search stops', async () => {
const config = makeConfig({
Expand Down Expand Up @@ -568,9 +564,9 @@ describe('5. Heuristic triggering conditions', () => {
const resolver = new HttpResolver(config);

// DNS fails for initial check, then resolves for all subsequent
mockedDnsResolve.mockRejectedValueOnce(new Error('ENOTFOUND') as never);
mockedDnsLookup.mockRejectedValueOnce({ address: '', family: 0 } as never);
// All subsequent DNS checks succeed
mockedDnsResolve.mockResolvedValue(['127.0.0.1'] as never);
mockedDnsLookup.mockResolvedValue({ address: '127.0.0.1', family: 4 } as never);

// resolver.resolve is only called for heuristic candidates (initial check fails at DNS)
jest.spyOn(resolver, 'resolve').mockImplementation(async (url: string) => {
Expand All @@ -590,7 +586,6 @@ describe('5. Heuristic triggering conditions', () => {
// ============================================================================
// 5.5 Content probe in heuristic
// ============================================================================

describe('5.5 Content probe in heuristic', () => {
test('probe_text present + body matches → candidate accepted', async () => {
const config = makeConfig({
Expand Down Expand Up @@ -644,7 +639,6 @@ describe('5.5 Content probe in heuristic', () => {
// ============================================================================
// 6.3 DNS pre-check
// ============================================================================

describe('6.3 DNS pre-check', () => {
test('dnsPreCheck.enabled: false → DNS check skipped, HTTP proceeds', async () => {
const config = makeConfig({ dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false } });
Expand All @@ -668,7 +662,7 @@ describe('6.3 DNS pre-check', () => {
const logger = makeLogger();
const resolver = new HttpResolver(config);

mockedDnsResolve.mockRejectedValueOnce(Object.assign(new Error('ENOTFOUND'), { code: 'ENOTFOUND' }) as never);
mockedDnsLookup.mockRejectedValueOnce(Object.assign(new Error('ENOTFOUND'), { code: 'ENOTFOUND' }) as never);

const processor = new BatchProcessor(config, watchers, logger, resolver);
const results = await processor.processAll();
Expand All @@ -681,7 +675,6 @@ describe('6.3 DNS pre-check', () => {
// ============================================================================
// Path mismatch
// ============================================================================

describe('Path mismatch handling', () => {
test('site.path set + final path differs → shouldUpdate: false, error message', async () => {
const config = makeConfig({ dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false } });
Expand All @@ -705,7 +698,6 @@ describe('Path mismatch handling', () => {
// ============================================================================
// No URL configured
// ============================================================================

describe('No URL configured', () => {
test('missing initial_domain and last_known_mirror → error result', async () => {
const config = makeConfig({ dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false } });
Expand All @@ -725,7 +717,6 @@ describe('No URL configured', () => {
// ============================================================================
// 7. skip_text scenarios
// ============================================================================

describe('7. skip_text scenarios', () => {
test('7.1 Scenario 1: main domain skipped, heuristic candidate OK → uses candidate', async () => {
const config = makeConfig({
Expand Down Expand Up @@ -895,7 +886,7 @@ describe('8. force_search_ahead scenarios', () => {
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 5, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: false,
});
Expand Down Expand Up @@ -931,7 +922,7 @@ describe('8. force_search_ahead scenarios', () => {
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 5, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: true,
});
Expand All @@ -941,7 +932,7 @@ describe('8. force_search_ahead scenarios', () => {

let callCount = 0;
const successfulCandidates: string[] = [];

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
if (callCount === 1) {
Expand Down Expand Up @@ -979,7 +970,7 @@ describe('8. force_search_ahead scenarios', () => {
dnsPreCheck: { enabled: false, timeout: 3000, retryOnce: false },
heuristic: { enabled: true, maxAttempts: 5, skipOnAntibot: true, forceHeuristicOnCodes: [] },
});
const site = makeSite({
const site = makeSite({
last_known_mirror: 'testsite1.com',
force_search_ahead: true,
probe_text: ['Expected Content'],
Expand All @@ -989,7 +980,7 @@ describe('8. force_search_ahead scenarios', () => {
const resolver = new HttpResolver(config);

let callCount = 0;

jest.spyOn(resolver, 'resolve').mockImplementation((url: string) => {
callCount++;
if (callCount === 1) {
Expand Down Expand Up @@ -1024,7 +1015,6 @@ describe('8. force_search_ahead scenarios', () => {
// ============================================================================
// 9. Antibot + force_search_ahead: heuristic should run even when accept_antibot succeeds
// ============================================================================

describe('9. Antibot + force_search_ahead + forceHeuristicOnCodes', () => {
test('9.1 accept_antibot + force_search_ahead: heuristic runs despite successful antibot check', async () => {
const config = makeConfig({
Expand Down Expand Up @@ -1383,7 +1373,6 @@ describe('9. Antibot + force_search_ahead + forceHeuristicOnCodes', () => {
// ============================================================================
// 10. probe_text filtering in heuristic search
// ============================================================================

describe('10. probe_text filtering in heuristic search', () => {
test('10.1 probe_text matches → heuristic succeeds', async () => {
const config = makeConfig({
Expand Down
66 changes: 33 additions & 33 deletions src/batch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ export class BatchProcessor {
* Tokenize domain into structured parts (runtime only, not persisted)
*/
private tokenizeDomain(domain: string): DomainToken {
const normalized = domain.startsWith('http://') || domain.startsWith('https://')
? domain
const normalized = domain.startsWith('http://') || domain.startsWith('https://')
? domain
: `https://${domain}`;
let hostname = this.resolver.extractHostWithoutQuery(normalized);
hostname = hostname.replace(/^www\./, '');
Expand Down Expand Up @@ -61,7 +61,7 @@ export class BatchProcessor {
return token.isPattern;
}


/**
* Group history domains by pattern (on the fly)
*/
Expand All @@ -88,7 +88,7 @@ export class BatchProcessor {
if (token.isPattern) {
// Pattern domain - reset flags (delete from config)
delete site.pattern_changed;

// Pattern → Pattern: DO NOT create history (just rotation)
// History is only needed when switching FROM pattern TO non-pattern
// So we delete any existing history when staying on pattern domains
Expand All @@ -100,7 +100,7 @@ export class BatchProcessor {
// Store only the last pattern domain before switching to non-pattern
site.heuristic_history = [oldLastKnownMirror];
}

site.pattern_changed = true;
}
}
Expand Down Expand Up @@ -133,7 +133,7 @@ export class BatchProcessor {
try {
const hostname = new URL(url).hostname;
await Promise.race([
dns.resolve(hostname),
dns.lookup(hostname),
new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), timeout))
]);
return true;
Expand All @@ -142,7 +142,7 @@ export class BatchProcessor {
try {
const hostname = new URL(url).hostname;
await Promise.race([
dns.resolve(hostname),
dns.lookup(hostname),
new Promise((_, rej) => setTimeout(() => rej(new Error("DNS timeout")), 2500))
]);
return true;
Expand All @@ -165,20 +165,20 @@ export class BatchProcessor {
failedUrl: string
): Promise<{ newHost: string; result: RedirectResult; candidateUrl: string } | null> {
const heuristicTasks = this.generateCandidates(siteName, siteIndex, site, failedUrl);

if (heuristicTasks.length === 0) {
this.logger.debug(siteName, `No heuristic candidates generated`);
return null;
}

const dnsCheckedTasks = await this.batchDnsCheck(heuristicTasks);
const dnsOkTasks = dnsCheckedTasks.filter(t => t.dnsOk);

if (dnsOkTasks.length === 0) {
this.logger.debug(siteName, `No heuristic candidates passed DNS check`);
return null;
}

return this.checkHeuristicCandidates(siteName, site, dnsOkTasks);
}

Expand All @@ -200,10 +200,10 @@ export class BatchProcessor {
this.logger.debug(siteName, `Heuristic: content probe is not configured for ${task.candidateUrl}`);
}
const httpResult = await this.resolver.resolve(task.candidateUrl, true, site, task.probeText);

if (httpResult.success) {
this.logger.debug(siteName, `Heuristic candidate ${task.candidateUrl}: HTTP ${httpResult.statusCode}${httpResult.antibotDetected ? ' (antibot)' : ''}`);

// Content probe if needed
let probeOk = true;
if (task.probeText && task.probeText.length > 0) {
Expand All @@ -219,17 +219,17 @@ export class BatchProcessor {
}
}
}

if (probeOk === true) {
const heuristicNewHost = httpResult.finalHost.toLowerCase();
const heuristicIsPattern = this.matchesNumericPattern(heuristicNewHost);

if (heuristicIsPattern) {
// Found a pattern domain!
const chainFormatted = this.resolver.formatRedirectChain(httpResult.redirectChain);
this.logger.info(siteName, `Heuristic SUCCESS: ${task.candidateUrl}`);
this.logger.info(siteName, `Heuristic redirect chain: ${chainFormatted}`);

return {
newHost: heuristicNewHost,
result: httpResult,
Expand All @@ -242,7 +242,7 @@ export class BatchProcessor {
}
}
}

return null; // No pattern domain found
}

Expand All @@ -260,20 +260,20 @@ export class BatchProcessor {
let match = failedUrl.match(/^(https?:\/\/)?(www\.)?([a-z-]+)(\d+)([a-z-]*)(\.[a-z.]+)(\/.*)?/i);
let isNumberFirst = false;
let wwwPrefix = '';

// Try pattern 2: [N]domain.tld (number at the beginning)
if (!match) {
match = failedUrl.match(/^(https?:\/\/)?(www\.)?(\d+)([a-z-]+)(\.[a-z.]+)(\/.*)?/i);
isNumberFirst = true;
}

if (!match) {
this.logger.warn(siteName, "Heuristic: URL doesn't match domain[N].tld, domain[N][text].tld, or [N]domain.tld pattern, skipping");
return [];
}

let protocol: string, prefix: string, numStr: string, middleText: string, suffix: string, path: string;

if (isNumberFirst) {
// Pattern: [N]domain.tld -> (protocol)(www.)(number)(letters)(suffix)(path)
[, protocol = 'https://', wwwPrefix = '', numStr, prefix, suffix, path = ''] = match;
Expand All @@ -282,7 +282,7 @@ export class BatchProcessor {
// Pattern: domain[N].tld or domain[N][text].tld -> (protocol)(www.)(letters)(number)(middle)(suffix)(path)
[, protocol = 'https://', wwwPrefix = '', prefix, numStr, middleText = '', suffix, path = ''] = match;
}

const currentNum = parseInt(numStr, 10);
const startNum = currentNum + 1;

Expand Down Expand Up @@ -437,21 +437,21 @@ export class BatchProcessor {
const candidates = this.generateCandidates(name, i, site, failedUrl);
allTasks.push(...candidates);
}

// Fallback: if site is working but on non-pattern domain, try history-based heuristic
if (!failed && !skipHeuristic && site.heuristic_history && site.heuristic_history.length > 0) {
const currentToken = this.tokenizeDomain(site.last_known_mirror || '');

if (!currentToken.isPattern) {
// Group history by pattern for smarter fallback
const patternGroups = this.groupHistoryByPattern(site.heuristic_history);

if (patternGroups.size > 0) {
this.logger.info(name, `Current domain ${currentToken.hostname} is non-pattern, trying history-based heuristic`);

for (const [patternKey, domains] of patternGroups) {
this.logger.info(name, `Trying pattern ${patternKey} with ${domains.length} domains`);

// Check ALL domains in this pattern group from first to last
for (const historyDomain of domains) {
// First, check the history domain itself
Expand All @@ -465,7 +465,7 @@ export class BatchProcessor {
site,
};
allTasks.push(historyTask);

// Then generate new candidates from the history domain
const candidates = this.generateCandidates(name, i, site, historyDomain);
allTasks.push(...candidates);
Expand Down Expand Up @@ -531,7 +531,7 @@ export class BatchProcessor {
// Atomic check-and-process: handle force_search_ahead differently
const alreadyFound = foundSites.has(task.siteIndex);
const continueSearch = task.site.force_search_ahead;

if (alreadyFound && !continueSearch) {
// Site already found and not force_search_ahead, skip processing
this.logger.debug(task.siteName, `Heuristic: skipping ${task.candidateUrl} (site already found)`);
Expand Down Expand Up @@ -710,7 +710,7 @@ export class BatchProcessor {
results[siteIndex].additionalWorkingDomains = domains
.map((d: { domain: string }) => d.domain)
.filter(domain => domain !== firstDomain);

this.logger.info(siteName, `force_search_ahead: collected ${domains.length} working domains: ${domains.map((d: { domain: string }) => d.domain).join(', ')}`);
}
}
Expand Down Expand Up @@ -983,18 +983,18 @@ export class BatchProcessor {
this.updateDomainHistory(site, newHost, site.last_known_mirror);
historyUpdated = true;
shouldUpdate = false; // do not touch filter files yet

// Try heuristic search immediately to find a new pattern domain
this.logger.info(siteName, `Triggering heuristic search to find new pattern domain...`);
const heuristicResult = await this.runHeuristicSearch(siteName, 0, site, oldHost);

if (heuristicResult) {
// Found a pattern domain via heuristic!
this.logger.info(siteName, `Heuristic found new pattern domain: ${heuristicResult.newHost}`);

// Update history: clear flags since we're back on pattern
this.updateDomainHistory(site, heuristicResult.newHost, oldHost);

// Return heuristic result instead
return {
siteName,
Expand Down