Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/commands/capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,8 @@ export const COMMAND_META: Record<string, CommandMeta> = {
'rules summary': READ_LOCAL,
'rules last-fired': READ_LOCAL,
'rules explain': READ_LOCAL,
'rules trace-explain': READ_LOCAL,
'rules simulate': READ_LOCAL,
'schema export': READ_LOCAL,
'scenes list': READ_REMOTE,
'scenes execute': ACTION_REMOTE,
Expand Down
8 changes: 5 additions & 3 deletions src/commands/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -934,11 +934,13 @@ function checkNotifyConnectivity(): Check {
return { name: 'notify-connectivity', status: 'ok', detail: { present: false, message: 'policy file could not be loaded' } };
}

const policy = loaded.data as { automation?: { rules?: Array<{ then?: Array<{ type?: string; channel?: string; to?: string }> }> } } | null;
const rules = policy?.automation?.rules ?? [];
const policy = loaded.data as { automation?: { rules?: unknown } } | null;
const rawRules = policy?.automation?.rules;
const rules = Array.isArray(rawRules) ? rawRules : [];
const webhookUrls: string[] = [];
for (const rule of rules) {
for (const action of rule.then ?? []) {
const then = (rule as { then?: unknown }).then;
for (const action of (Array.isArray(then) ? then : []) as Array<{ type?: string; channel?: string; to?: string }>) {
if (action.type === 'notify' && (action.channel === 'webhook' || action.channel === 'openclaw') && action.to) {
webhookUrls.push(action.to);
}
Expand Down
152 changes: 152 additions & 0 deletions src/commands/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ import { planMigration } from '../policy/migrate.js';
import { suggestPlan } from './plan.js';
import { suggestRule } from '../rules/suggest.js';
import { addRuleToPolicyFile, AddRuleError } from '../policy/add-rule.js';
import {
loadTraceRecords,
loadRelatedAudit,
formatExplainJson,
} from '../rules/explain.js';
import { simulateRule } from '../rules/simulate.js';
import { allowsDirectDestructiveExecution, destructiveExecutionHint } from '../lib/destructive-mode.js';
import { writeFileSync } from 'node:fs';
import { readAudit, type AuditEntry } from '../utils/audit.js';
Expand Down Expand Up @@ -1972,6 +1978,152 @@ API docs: https://github.com/OpenWonderLabs/SwitchBotAPI`,
},
);

// ---- rules_explain --------------------------------------------------------
server.registerTool(
'rules_explain',
{
title: 'Show why a rule evaluation fired or was blocked',
description:
'Read rule-evaluate trace records from the audit log and format them for inspection. ' +
'Pass fire_id to explain a specific evaluation; or pass rule_name with last:true for the ' +
'most recent evaluation; or pass rule_name + since for a window. ' +
'Returns trace records only when automation.audit.evaluate_trace is "sampled" or "full".',
_meta: { agentSafetyTier: 'read' },
inputSchema: z.object({
fire_id: z.string().optional().describe('Specific fireId to explain.'),
rule_name: z.string().optional().describe('Filter to this rule name.'),
since: z.string().optional().describe('Duration string (e.g. 1h, 7d) — show evaluations in this window.'),
last: z.boolean().optional().describe('Return only the most recent evaluation (requires rule_name).'),
audit_log: z.string().optional().describe(`Audit log path (default: ${pathJoin(os.homedir(), '.switchbot', 'audit.log')}).`),
}).strict(),
outputSchema: {
records: z.array(z.unknown()).describe('Array of trace + relatedAudit objects.'),
count: z.number().describe('Number of trace records returned.'),
},
},
async ({ fire_id, rule_name, since, last, audit_log }) => {
const DEFAULT_AUDIT_PATH = pathJoin(os.homedir(), '.switchbot', 'audit.log');
const auditFile = audit_log ?? DEFAULT_AUDIT_PATH;
const sinceIso = since
? new Date(Date.now() - (parseDurationToMs(since) ?? 0)).toISOString()
: undefined;

let records = loadTraceRecords(auditFile, {
fireId: fire_id,
ruleName: rule_name,
since: sinceIso,
});

if (records.length === 0) {
return {
content: [{ type: 'text' as const, text: 'No rule-evaluate trace records found. Check that automation.audit.evaluate_trace is "sampled" or "full".' }],
structuredContent: { records: [], count: 0 },
};
}

if (last) {
records = [records[records.length - 1]];
}

const output = records.map((record) => {
const related = loadRelatedAudit(auditFile, record.fireId);
return JSON.parse(formatExplainJson(record, related)) as unknown;
});

return {
content: [{ type: 'text' as const, text: JSON.stringify(output, null, 2) }],
structuredContent: { records: output, count: output.length },
};
},
);

// ---- rules_simulate -------------------------------------------------------
server.registerTool(
'rules_simulate',
{
title: 'Simulate a rule against historical events',
description:
'Replay historical events from the audit log or a JSONL file against a rule definition ' +
'and report would-fire / blocked-by-condition / throttled outcomes. ' +
'Useful for validating a new or modified rule before deployment. ' +
'Pass rule_yaml to test an unpublished rule, or rule_name + policy_path to test a deployed rule.',
_meta: { agentSafetyTier: 'read' },
inputSchema: z.object({
rule_yaml: z.string().optional().describe('Standalone rule YAML (takes precedence over policy_path + rule_name).'),
policy_path: z.string().optional().describe('Path to policy.yaml (defaults to ~/.switchbot/policy.yaml).'),
rule_name: z.string().optional().describe('Name of the rule in policy.yaml to simulate.'),
since: z.string().optional().describe('Replay events from this window (e.g. 7d, 24h).'),
against: z.string().optional().describe('JSONL file path of EngineEvent objects to replay.'),
live_llm: z.boolean().optional().describe('Allow live LLM calls for llm conditions (default: skip and report as would-call).'),
audit_log: z.string().optional().describe(`Audit log path (default: ${pathJoin(os.homedir(), '.switchbot', 'audit.log')}).`),
}).strict(),
outputSchema: {
report: z.unknown().describe('SimulateReport object.'),
},
},
async ({ rule_yaml, policy_path, rule_name, since, against, live_llm, audit_log }) => {
const DEFAULT_AUDIT_PATH = pathJoin(os.homedir(), '.switchbot', 'audit.log');
const auditFile = audit_log ?? DEFAULT_AUDIT_PATH;

let rule: Record<string, unknown> | undefined;

if (rule_yaml) {
try {
rule = yamlParse(rule_yaml) as Record<string, unknown>;
} catch (err) {
return {
content: [{ type: 'text' as const, text: `Failed to parse rule_yaml: ${String(err)}` }],
structuredContent: { report: null },
};
}
} else if (policy_path || rule_name) {
const { loadPolicyFile } = await import('../policy/load.js');
const policyFile = policy_path ?? pathJoin(os.homedir(), '.switchbot', 'policy.yaml');
try {
const policy = loadPolicyFile(policyFile);
const data = (policy.data ?? {}) as { automation?: { rules?: Array<{ name: string }> } };
const found = data.automation?.rules?.find((r) => r.name === rule_name);
if (!found) {
return {
content: [{ type: 'text' as const, text: `Rule "${rule_name}" not found in ${policyFile}.` }],
structuredContent: { report: null },
};
}
rule = found as unknown as Record<string, unknown>;
} catch (err) {
return {
content: [{ type: 'text' as const, text: `Failed to load policy: ${String(err)}` }],
structuredContent: { report: null },
};
}
} else {
return {
content: [{ type: 'text' as const, text: 'Provide rule_yaml or (policy_path + rule_name) to specify the rule to simulate.' }],
structuredContent: { report: null },
};
}

try {
const report = await simulateRule({
rule: rule as unknown as Parameters<typeof simulateRule>[0]['rule'],
since,
against,
auditLog: auditFile,
liveLlm: live_llm ?? false,
});
return {
content: [{ type: 'text' as const, text: JSON.stringify(report, null, 2) }],
structuredContent: { report },
};
} catch (err) {
return {
content: [{ type: 'text' as const, text: `Simulate error: ${String(err)}` }],
structuredContent: { report: null },
};
}
},
);

// ---- policy_add_rule ------------------------------------------------------
server.registerTool(
'policy_add_rule',
Expand Down
157 changes: 157 additions & 0 deletions src/commands/rules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@ import {
isPidAlive,
} from '../rules/pid-file.js';
import { readAudit, type AuditEntry } from '../utils/audit.js';
import {
loadTraceRecords,
loadRelatedAudit,
formatExplainText,
formatExplainJson,
} from '../rules/explain.js';
import { simulateRule } from '../rules/simulate.js';
import {
aggregateRuleAudits,
filterRuleAudits,
Expand Down Expand Up @@ -892,6 +899,154 @@ function registerExplain(rules: Command): void {
});
}

function registerTraceExplain(rules: Command): void {
rules
.command('trace-explain [fireId]')
.description('Show why a rule evaluation fired or was blocked (reads rule-evaluate trace records).')
.option('--rule <name>', 'Filter to a specific rule name.')
.option('--last', 'Show the most recent evaluation for the rule (requires --rule).')
.option('--since <duration>', 'Show evaluations in this window (e.g. 1h, 7d).')
.option('--all', 'Include evaluations that fired (default: show all evaluations).')
.option('--file <path>', `Audit log path (default ${DEFAULT_AUDIT_PATH}).`)
.action(
(
fireIdArg: string | undefined,
opts: { rule?: string; last?: boolean; since?: string; all?: boolean; file?: string },
) => {
const auditFile = opts.file ?? DEFAULT_AUDIT_PATH;

if (!fs.existsSync(auditFile)) {
exitWithError({ code: 1, kind: 'usage', message: `Audit log not found: ${auditFile}. Make sure trace recording is enabled (automation.audit.evaluate_trace: sampled or full).` });
return;
}

const sinceIso = opts.since ? new Date(Date.now() - (parseDurationToMs(opts.since) ?? 0)).toISOString() : undefined;

let records = loadTraceRecords(auditFile, {
fireId: fireIdArg,
ruleName: opts.rule,
since: sinceIso,
});

if (records.length === 0) {
const hint = 'Check that automation.audit.evaluate_trace is set to "sampled" or "full".';
exitWithError({ code: 1, kind: 'usage', message: `No rule-evaluate trace records found. ${hint}` });
return;
}

if (opts.last) {
records = [records[records.length - 1]];
}

for (const record of records) {
const related = loadRelatedAudit(auditFile, record.fireId);
if (isJsonMode()) {
console.log(formatExplainJson(record, related));
} else {
console.log(formatExplainText(record, related));
if (records.length > 1) console.log('---');
}
}
},
);
}

function registerSimulate(rules: Command): void {
rules
.command('simulate <rule-or-policy>')
.description('Replay historical events against a rule and report would-fire / blocked outcomes.')
.option('--rule <name>', 'Rule name to simulate (when <rule-or-policy> is a policy file).')
.option('--since <duration>', 'Replay events from this window (e.g. 7d, 24h).')
.option('--against <file>', 'Replay from a JSONL file of EngineEvent objects instead of the audit log.')
.option('--live-llm', 'Allow live LLM calls for llm conditions (default: mark as would-call).')
.option('--audit-log <path>', `Audit log path (default ${DEFAULT_AUDIT_PATH}).`)
.option('--report-out <path>', 'Write the full JSON report to this file.')
.action(
async (
ruleOrPolicy: string,
opts: { rule?: string; since?: string; against?: string; liveLlm?: boolean; auditLog?: string; reportOut?: string },
) => {
let rule: Rule | undefined;
const auditLog = opts.auditLog ?? DEFAULT_AUDIT_PATH;

if (!fs.existsSync(ruleOrPolicy)) {
exitWithError({ code: 2, kind: 'usage', message: `File not found: ${ruleOrPolicy}` });
return;
}

// Try to parse as standalone rule YAML
let parsed: unknown;
try {
const { parse: yamlParse } = await import('yaml');
parsed = yamlParse(fs.readFileSync(ruleOrPolicy, 'utf-8'));
} catch {
exitWithError({ code: 2, kind: 'usage', message: `Could not parse YAML file: ${ruleOrPolicy}` });
return;
}

const asRule = parsed as Record<string, unknown>;
if (asRule['name'] && asRule['when'] && asRule['then']) {
rule = asRule as unknown as Rule;
} else {
// Treat as policy file
const automation = loadAutomation(ruleOrPolicy);
if (!automation) return;
const ruleName = opts.rule;
if (!ruleName) {
exitWithError({ code: 1, kind: 'usage', message: 'Use --rule <name> to specify which rule to simulate from the policy file.' });
return;
}
rule = automation.automation?.rules?.find(r => r.name === ruleName);
if (!rule) {
exitWithError({ code: 1, kind: 'usage', message: `Rule "${ruleName}" not found in policy file.` });
return;
}
}

try {
const report = await simulateRule({
rule: rule!,
since: opts.since,
against: opts.against,
auditLog,
liveLlm: opts.liveLlm ?? false,
});

if (opts.reportOut) {
fs.writeFileSync(opts.reportOut, JSON.stringify(report, null, 2));
console.log(`Report written to ${opts.reportOut}`);
}

if (isJsonMode()) {
printJson(report);
} else {
console.log(`Rule: ${report.ruleName} (version ${report.ruleVersion})`);
console.log(`Window: ${report.windowStart.toISOString()} → ${report.windowEnd.toISOString()}`);
console.log(`Source events: ${report.sourceEventCount}`);
console.log('');
console.log(` Would fire: ${report.wouldFire}`);
console.log(` Blocked by condition:${report.blockedByCondition}`);
console.log(` Throttled: ${report.throttled}`);
console.log(` Errored: ${report.errored}`);
if (report.skippedLlm > 0) {
console.log(` Skipped (llm): ${report.skippedLlm} (use --live-llm to evaluate)`);
}
if (report.topBlockReason && report.topBlockCount !== undefined) {
const total = report.blockedByCondition;
const pct = total > 0 ? Math.round((report.topBlockCount / total) * 100) : 0;
if (pct >= 80) {
console.log('');
console.log(`Top block reason (${pct}%): ${report.topBlockReason}`);
}
}
}
} catch (err) {
handleError(err);
}
},
);
}

export function registerRulesCommand(program: Command): void {
const rules = program
.command('rules')
Expand Down Expand Up @@ -942,6 +1097,8 @@ Exit codes (lint):
registerDoctor(rules);
registerSummary(rules);
registerLastFired(rules);
registerTraceExplain(rules);
registerSimulate(rules);
registerWebhookRotateToken(rules);
registerWebhookShowToken(rules);
}
10 changes: 10 additions & 0 deletions src/llm/provider.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
export interface DecideResult {
pass: boolean;
reason: string;
}

export interface DecideOptions {
timeoutMs?: number;
}

export interface LLMProvider {
readonly name: string;
readonly model: string;
generateYaml(systemPrompt: string, userIntent: string): Promise<string>;
decide(prompt: string, opts?: DecideOptions): Promise<DecideResult>;
}

export interface LLMProviderOptions {
Expand Down
Loading
Loading