audit.test.js

/**
 * Tests for SkillGuard Audit Script (JavaScript)
 * Uses a local mock HTTP server to simulate the SkillGuard API.
 *
 * Run with: node audit.test.js
 */

const http = require('http');
const assert = require('assert');
const { spawn } = require('child_process');
const path = require('path');

const AUDIT_JS = path.join(__dirname, 'audit.js');

// ──────────────────────────────────────────────
//  Mock HTTP Server
// ──────────────────────────────────────────────
function createMockServer(scenario) {
  let pollCount = 0;

  return http.createServer((req, res) => {
    const url = req.url;
    let body = '';
    req.on('data', (chunk) => (body += chunk));
    req.on('end', () => {
      res.setHeader('Content-Type', 'application/json');

      // POST /api/scan
      if (req.method === 'POST' && url === '/api/scan') {
        if (scenario === 'scan_error') {
          res.statusCode = 500;
          res.end('Internal Server Error');
          return;
        }
        res.end(JSON.stringify({ scan_id: 'scan-001' }));
        return;
      }

      // GET /api/scan/:id/status
      if (req.method === 'GET' && url.match(/^\/api\/scan\/[^/]+\/status$/)) {
        pollCount++;
        if (scenario === 'scan_status_error') {
          res.end(JSON.stringify({ status: 'error', error: 'Backend crashed' }));
          return;
        }
        if (pollCount < 2) {
          res.end(JSON.stringify({ status: 'processing', progress: 50 }));
        } else {
          res.end(JSON.stringify({ status: 'done', progress: 100 }));
        }
        return;
      }

      // GET /api/report/:id
      if (req.method === 'GET' && url.match(/^\/api\/report\/[^/]+$/)) {
        if (scenario === 'grade_c') {
          res.end(JSON.stringify({
            skill_name: 'risky-skill',
            risk_level: 'C',
            risk_score: 62,
            findings: [
              { severity: 'CRITICAL', dimension: 'Prompt Injection', description: 'Direct prompt injection found in system message handler function' },
              { severity: 'HIGH', dimension: 'Data Exfiltration', description: 'Outbound POST to untrusted external endpoint with user data' },
              { severity: 'MEDIUM', dimension: 'Code Execution', description: 'Unrestricted eval usage with concatenated input' },
              { severity: 'LOW', dimension: 'File System', description: 'Reads config file from relative path' },
              { severity: 'LOW', dimension: 'Credential Handling', description: 'API key stored in local variable' }
            ]
          }));
        } else if (scenario === 'no_findings') {
          res.end(JSON.stringify({
            skill_name: 'safe-skill',
            risk_level: 'A',
            risk_score: 5,
            findings: []
          }));
        } else {
          res.end(JSON.stringify({
            skill_name: 'test-skill',
            risk_level: 'B',
            risk_score: 30,
            findings: [
              { severity: 'MEDIUM', dimension: 'Network', description: 'HTTP request to user-supplied URL without validation' },
              { severity: 'LOW', dimension: 'Resource Abuse', description: 'No rate limiting on API calls' }
            ]
          }));
        }
        return;
      }

      // POST /api/deep-scan
      if (req.method === 'POST' && url === '/api/deep-scan') {
        const payload = JSON.parse(body);
        if (!payload.api_key) {
          res.statusCode = 400;
          res.end('API key required');
          return;
        }
        res.end(JSON.stringify({ deep_scan_id: 'ds-001' }));
        return;
      }

      // GET /api/deep-scan/:id/status
      if (req.method === 'GET' && url.match(/^\/api\/deep-scan\/[^/]+\/status$/)) {
        res.end(JSON.stringify({ status: 'done', phase: 'complete', progress: 100 }));
        return;
      }

      // GET /api/deep-scan/:id/report
      if (req.method === 'GET' && url.match(/^\/api\/deep-scan\/[^/]+\/report$/)) {
        res.end(JSON.stringify({
          risk_score: 68,
          total_turns: 12,
          total_tool_calls: 34,
          evidences: [
            { risk_level: 'HIGH', description: 'Skill executed bash command with unsanitized user input' },
            { risk_level: 'MEDIUM', description: 'File read operation accessed sensitive config' }
          ],
          actual_cost: 0.1234
        }));
        return;
      }

      res.statusCode = 404;
      res.end('Not found');
    });
  });
}

function startServer(scenario) {
  return new Promise((resolve) => {
    const server = createMockServer(scenario);
    server.listen(0, '127.0.0.1', () => {
      const port = server.address().port;
      resolve({ server, port });
    });
  });
}

function stopServer(server) {
  return new Promise((resolve) => server.close(resolve));
}

function runAudit(url, port, input) {
  return new Promise((resolve) => {
    const proc = spawn('node', [AUDIT_JS, url], {
      env: { ...process.env, SKILLGUARD_API: `http://127.0.0.1:${port}` },
      stdio: ['pipe', 'pipe', 'pipe'],
      cwd: __dirname
    });

    let stdout = '';
    let stderr = '';

    proc.stdout.on('data', (d) => (stdout += d));
    proc.stderr.on('data', (d) => (stderr += d));

    if (input !== undefined) {
      proc.stdin.write(input);
      proc.stdin.end();
    }

    proc.on('close', (code) => resolve({ code, stdout, stderr }));
  });
}

// ──────────────────────────────────────────────
//  Tests
// ──────────────────────────────────────────────
let passed = 0;
let failed = 0;

function ok(condition, msg) {
  if (condition) {
    console.log(`  ✓ ${msg}`);
    passed++;
  } else {
    console.log(`  ✗ ${msg}`);
    failed++;
  }
}

async function test_missingArg() {
  console.log('\n[1] Missing URL argument');
  const proc = spawn('node', [AUDIT_JS], { stdio: ['pipe', 'pipe', 'pipe'] });
  let stdout = '';
  proc.stdout.on('data', (d) => (stdout += d));
  const code = await new Promise((r) => proc.on('close', r));
  ok(code === 1, 'exits with code 1');
  ok(stdout.includes('Usage:'), 'prints usage message');
}

async function test_normalScan() {
  console.log('\n[2] Normal scan (grade B)');
  const { server, port } = await startServer('default');
  const result = await runAudit('https://github.com/user/test-skill', port);
  await stopServer(server);

  ok(result.code === 0, 'exits with code 0');
  ok(result.stdout.includes('Submitting scan'), 'shows submitting message');
  ok(result.stdout.includes('Scan ID: scan-001'), 'shows scan ID');
  ok(result.stdout.includes('SkillGuard Security Report'), 'shows report header');
  ok(result.stdout.includes('Skill: test-skill'), 'shows skill name');
  ok(result.stdout.includes('Grade: B'), 'shows grade');
  ok(result.stdout.includes('Risk Score: 30/100'), 'shows risk score');
  ok(result.stdout.includes('Total Findings: 2'), 'shows finding count');
  ok(result.stdout.includes('MEDIUM: 1'), 'shows risk breakdown');
  ok(result.stdout.includes('LOW: 1'), 'shows risk breakdown (LOW)');
  ok(result.stdout.includes('Network'), 'shows finding dimension');
}

async function test_scanWithPolling() {
  console.log('\n[3] Scan with status polling');
  const { server, port } = await startServer('default');
  const result = await runAudit('https://github.com/user/poll-test', port);
  await stopServer(server);

  ok(result.stdout.includes('Waiting for results'), 'shows waiting message');
  ok(result.stdout.includes('Status: processing (50%)'), 'shows intermediate status');
}

async function test_noFindings() {
  console.log('\n[4] Clean skill with no findings (grade A)');
  const { server, port } = await startServer('no_findings');
  const result = await runAudit('https://github.com/user/safe-skill', port);
  await stopServer(server);

  ok(result.code === 0, 'exits with code 0');
  ok(result.stdout.includes('Grade: A'), 'shows grade A');
  ok(result.stdout.includes('Risk Score: 5/100'), 'shows low risk score');
  ok(result.stdout.includes('Total Findings: 0'), 'shows zero findings');
  ok(!result.stdout.includes('Top Findings'), 'does not show top findings section');
  ok(!result.stdout.includes('Deep Scan recommended'), 'does not offer deep scan');
}

async function test_gradeCDeepScanDecline() {
  console.log('\n[5] Grade C - decline deep scan');
  const { server, port } = await startServer('grade_c');
  const result = await runAudit('https://github.com/user/risky-skill', port, 'N\n');
  await stopServer(server);

  ok(result.stdout.includes('Grade: C'), 'shows grade C');
  ok(result.stdout.includes('Risk Score: 62/100'), 'shows risk score');
  ok(result.stdout.includes('CRITICAL: 1'), 'shows critical count');
  ok(result.stdout.includes('HIGH: 1'), 'shows high count');
  ok(result.stdout.includes('Deep Scan recommended'), 'offers deep scan');
  ok(!result.stdout.includes('Deep Scan Report'), 'does not run deep scan');
}

async function test_scanSubmitError() {
  console.log('\n[6] API error on scan submit');
  const { server, port } = await startServer('scan_error');
  const result = await runAudit('https://github.com/user/error-test', port);
  await stopServer(server);

  ok(result.code === 1, 'exits with code 1');
  ok(result.stderr.includes('Error'), 'shows error message');
}

async function test_scanStatusError() {
  console.log('\n[7] Error during scan status polling');
  const { server, port } = await startServer('scan_status_error');
  const result = await runAudit('https://github.com/user/status-error', port);
  await stopServer(server);

  ok(result.code === 1, 'exits with code 1');
  ok(result.stderr.includes('Scan failed'), 'shows scan failed message');
  ok(result.stderr.includes('Backend crashed'), 'shows error detail');
}

async function test_findingsTruncation() {
  console.log('\n[8] Findings description truncation');
  const { server, port } = await startServer('grade_c');
  const result = await runAudit('https://github.com/user/truncate-test', port, 'N\n');
  await stopServer(server);

  // Findings shown in top section should have trailing '...'
  ok(result.stdout.includes('...'), 'truncates findings with ellipsis');
  // Only top 5 findings displayed
  ok(result.stdout.includes('Total Findings: 5'), 'shows correct total');
}

async function test_fullReportURL() {
  console.log('\n[9] Full report URL');
  const { server, port } = await startServer('default');
  const result = await runAudit('https://github.com/user/url-test', port);
  await stopServer(server);

  ok(result.stdout.includes(`http://127.0.0.1:${port}/report/scan-001`), 'shows full report URL');
}

// ──────────────────────────────────────────────
//  Runner
// ──────────────────────────────────────────────
async function main() {
  console.log('='.repeat(60));
  console.log('  SkillGuard Audit Script Tests (JavaScript)');
  console.log('='.repeat(60));

  await test_missingArg();
  await test_normalScan();
  await test_scanWithPolling();
  await test_noFindings();
  await test_gradeCDeepScanDecline();
  await test_scanSubmitError();
  await test_scanStatusError();
  await test_findingsTruncation();
  await test_fullReportURL();

  console.log('\n' + '='.repeat(60));
  console.log(`  Results: ${passed} passed, ${failed} failed`);
  console.log('='.repeat(60));

  if (failed > 0) process.exit(1);
}

main().catch((err) => {
  console.error('Fatal:', err);
  process.exit(1);
});