From 1d41eff88009433f5af40edaeb65183678158c5f Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:50:28 -0400 Subject: [PATCH 01/10] feat(storage): storage doctor, ingress payload elimination, and Storage tab health surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root-caused the 2026-07-17 daemon wedge: unbounded data accumulation (450MB of never-read webhook payloads — 32,023 of 32,026 events ignored), DELETE-mode journaling, and zero automatic maintenance. - automations: never persist raw webhook payloads (verified never read); write-time retention on ingress events (7d / 2,000 per project); one-time boot reclaim nulls legacy payloads, prunes stale review artifacts + PR snapshots (rehearsed on the real 614MB DB: -> 146MB in ~1.6s) - kvDb: WAL + synchronous=NORMAL on rw opens; DbMaintenanceApi on the handle (ingress/artifact/snapshot pruning, peer-gated crsql compaction, fragmentation-aware VACUUM that activates incremental auto_vacuum) - storage doctor: existing daemon sweep extended into a per-step-isolated maintenance pass (transcript compression 30d->14d, auto-reap of os.tmpdir + .ade/tmp staging, obsolete recovery backups, DerivedData), 30-run journal, storage ledger with CI coverage test, runMaintenanceNow wired through action-domain/IPC/preload, 24h slow-action counter via getRuntimeHealth, storage_doctor PostHog taxonomy - Storage tab: SettingsSectionShell header, one-click safe cleanup with itemized preview, policy chips, DB internal breakdown replacing the "Protected" black box, diagnostics strip (DB trend, daemon memory, slow responses, last cleanup), maintenance journal, load pill now deep-links to settings diagnostics Co-Authored-By: Claude Fable 5 --- apps/ade-cli/src/bootstrap.ts | 6 + apps/desktop/src/main/main.ts | 11 +- .../src/main/services/adeActions/registry.ts | 5 +- .../analytics/productAnalyticsPolicy.ts | 13 +- .../analytics/productAnalyticsService.test.ts | 30 + .../automations/automationService.test.ts | 145 ++++ .../services/automations/automationService.ts | 151 +++- .../src/main/services/ipc/registerIpc.ts | 17 + .../localRuntimeConnectionPool.test.ts | 29 + .../localRuntimeConnectionPool.ts | 46 +- .../main/services/state/dbMaintenanceApi.ts | 35 + .../state/kvDb.rebuildRecovery.test.ts | 135 +++ apps/desktop/src/main/services/state/kvDb.ts | 191 ++++- .../services/storage/historyCompression.ts | 6 +- .../storage/storageInsightsService.test.ts | 242 +++++- .../storage/storageInsightsService.ts | 466 +++++++++- .../services/storage/storageLedger.test.ts | 73 ++ .../main/services/storage/storageLedger.ts | 208 +++++ apps/desktop/src/preload/global.d.ts | 4 + apps/desktop/src/preload/preload.ts | 8 + apps/desktop/src/renderer/browserMock.ts | 53 +- .../renderer/components/app/SettingsPage.tsx | 1 + .../src/renderer/components/app/TopBar.tsx | 9 +- .../settings/StorageSection.test.tsx | 176 +++- .../components/settings/StorageSection.tsx | 804 +++++++++++++++--- .../settings/storage/StorageCleanupDialog.tsx | 197 ++++- .../settings/storage/storageView.test.ts | 276 ++++++ .../settings/storage/storageView.ts | 395 +++++++++ apps/desktop/src/shared/ipc.ts | 2 + apps/desktop/src/shared/types/storage.ts | 84 ++ docs/logging.md | 2 + 31 files changed, 3617 insertions(+), 203 deletions(-) create mode 100644 apps/desktop/src/main/services/state/dbMaintenanceApi.ts create mode 100644 apps/desktop/src/main/services/storage/storageLedger.test.ts create mode 100644 apps/desktop/src/main/services/storage/storageLedger.ts create mode 100644 apps/desktop/src/renderer/components/settings/storage/storageView.test.ts diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index 680e4a41e..7c2fb3651 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -1504,6 +1504,12 @@ export async function createAdeRuntime(args: { isPathActive: (filePath) => Boolean(agentChatService?.isTranscriptPathActive(filePath)) || ptyService.isTranscriptPathActive(filePath), + projectId, + // One bounded `ade_feature_used` per completed maintenance run at the daemon + // boundary (deduped to 20 h by the service). + captureAnalytics: (input) => { + productAnalyticsService.capture(input); + }, }); const budgetCapService = createBudgetCapService({ db, diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 900058d6b..0515a9ef6 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -3587,6 +3587,10 @@ app.whenReady().then(async () => { isPathActive: (filePath) => agentChatService.isTranscriptPathActive(filePath) || ptyService.isTranscriptPathActive(filePath), + projectId, + captureAnalytics: (input) => { + productAnalyticsService.capture(input); + }, }); // Phone sync is owned by the per-machine ADE service. The desktop @@ -4323,8 +4327,13 @@ app.whenReady().then(async () => { logger, // Daemon-backed mode: the brain owns activity tracking and runs the real compression sweep // (see apps/ade-cli/src/bootstrap.ts); this fallback instance deliberately refuses to compress - // because activity cannot be known here. + // because activity cannot be known here. It also never arms the maintenance timers (no + // diskPressure supplied), so it only runs maintenance if runMaintenanceNow is called directly. isPathActive: () => true, + projectId: runtimeProject.projectId, + captureAnalytics: (input) => { + productAnalyticsService.capture(input); + }, }); const diskPressureMonitor = createDiskPressureMonitor({ roots: [projectRoot, machineAdeLayout.adeDir], diff --git a/apps/desktop/src/main/services/adeActions/registry.ts b/apps/desktop/src/main/services/adeActions/registry.ts index 3dd9131d9..ab6600b13 100644 --- a/apps/desktop/src/main/services/adeActions/registry.ts +++ b/apps/desktop/src/main/services/adeActions/registry.ts @@ -173,7 +173,7 @@ export const ADE_ACTION_CTO_ONLY: Partial storageInsightsService.getSnapshot(args), compressNow: () => storageInsightsService.compressNow(), + runMaintenanceNow: () => storageInsightsService.runMaintenanceNow(), cleanupPreview: (args?: { targets?: Parameters[0] }) => storageInsightsService.cleanupPreview(args?.targets ?? []), cleanup: (args?: { diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts index a0c5ab104..3fc1d3454 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts @@ -50,10 +50,15 @@ const NUMBER_PROPERTIES = new Set([ "terminal_session_count", "active_lane_count", "lanes_created", "lanes_archived", "commits_created", "push_operations", "pr_landings", "files_changed", "artifacts_captured", "automation_runs", "worker_runs", "active_days", "current_streak_days", "token_count", "input_token_count", "output_token_count", "call_count", - "duration_ms", "provider_count", "model_count", "error_count", + "duration_ms", "provider_count", "model_count", "error_count", "bytes_freed", "files_compressed", ]); const BOOLEAN_PROPERTIES = new Set(["recoverable", "paired", "cached_data", "is_packaged"]); +// Actions emitted only by daemon services (not user-mutation ledger rows) that +// are still meaningful product facts. Kept here rather than in the usage-stats +// MEANINGFUL_ACTIONS set because they never correspond to a persisted mutation. +const ANALYTICS_ONLY_ACTIONS = new Set(["maintenance_run"]); + const EVENT_PROPERTY_KEYS: Record> = { ade_app_opened: new Set([ "entry_point", "source", "release_channel", "mode", "connection_state", "paired", "cached_data", "is_packaged", @@ -62,6 +67,7 @@ const EVENT_PROPERTY_KEYS: Record ade_project_opened: new Set(["route_kind", "source", "mode", "connection_state"]), ade_feature_used: new Set([ "feature", "action", "outcome", "source", "mode", "provider", "model_family", "duration_bucket", "connection_state", + "bytes_freed", "files_compressed", ]), ade_work_session_started: new Set(["feature", "action", "outcome", "source", "mode", "provider"]), ade_work_session_completed: new Set([ @@ -89,10 +95,11 @@ const SAFE_STRING_VALUES: Partial>> = { ]), feature: new Set([ "chat", "cli", "work", "lanes", "files", "git", "processes", "orchestration", "prs", - "automations", "command_palette", + "automations", "command_palette", "storage_doctor", ]), outcome: new Set([ "success", "started", "completed", "failure", "timeout", "opened", "cancelled", "approved", "denied", + "partial", "failed", ]), provider: new Set(["codex", "openai", "claude", "cursor", "droid", "opencode", "gemini", "local", "other"]), model_family: new Set([ @@ -141,7 +148,7 @@ function safeStringProperty(key: string, value: ProductAnalyticsPropertyValue): if (key === "action") { if (typeof value !== "string" || value.length > 256) return null; const raw = value.trim(); - return raw === "open" || isMeaningfulUsageAction(raw) ? raw : null; + return raw === "open" || isMeaningfulUsageAction(raw) || ANALYTICS_ONLY_ACTIONS.has(raw) ? raw : null; } if (key === "error_kind") return coarseErrorKind(value); const safe = safeProductAnalyticsString(value); diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts index 02cb43b49..6243478b6 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts @@ -136,6 +136,36 @@ describe("productAnalyticsService", () => { fs.rmSync(harness.root, { recursive: true, force: true }); }); + it("accepts the storage-doctor maintenance event with numeric aggregates and coarse outcome", () => { + const harness = makeHarness(); + const result = harness.service.capture({ + event: "ade_feature_used", + surface: "desktop", + properties: { + feature: "storage_doctor", + action: "maintenance_run", + outcome: "partial", + bytes_freed: 481_000_000, + files_compressed: 62, + secret_path: "/Users/alice/secret-project/.ade", + }, + }); + + expect(result).toEqual({ accepted: true, reason: "accepted" }); + const message = harness.messages[0] as { properties: Record }; + expect(message.properties).toMatchObject({ + feature: "storage_doctor", + action: "maintenance_run", + outcome: "partial", + bytes_freed: 481_000_000, + files_compressed: 62, + }); + // Non-allowlisted keys never cross the sanitizer. + expect(message.properties).not.toHaveProperty("secret_path"); + expect(JSON.stringify(message)).not.toContain("secret-project"); + fs.rmSync(harness.root, { recursive: true, force: true }); + }); + it("does not forward arbitrary build-controlled version text", () => { const harness = makeHarness({ appVersion: "../../private/project\nsecret" }); expect(harness.service.capture({ diff --git a/apps/desktop/src/main/services/automations/automationService.test.ts b/apps/desktop/src/main/services/automations/automationService.test.ts index a9742a3e0..24ac97964 100644 --- a/apps/desktop/src/main/services/automations/automationService.test.ts +++ b/apps/desktop/src/main/services/automations/automationService.test.ts @@ -2445,3 +2445,148 @@ describe("buildLinearAutomationDispatches", () => { expect(dispatches.map((d) => d.triggerType)).toEqual(["linear.issue_updated"]); }); }); + +describe("automation ingress storage bounds", () => { + function createIngressService( + db: AdeDb, + logger = createLogger(), + ): ReturnType { + return createAutomationService({ + db: db as any, + logger, + projectId: "proj", + projectRoot: "/tmp", + laneService: { + list: async () => [], + getLaneBaseAndBranch: () => ({ baseRef: "main", branchRef: "main", worktreePath: "/tmp" }), + getLaneWorktreePath: () => "/tmp", + } as any, + projectConfigService: { + get: () => ({ + trust: { requiresSharedTrust: false }, + effective: { automations: [], providerMode: "guest" }, + }), + } as any, + }); + } + + it("keeps only 2,000 slim rows during an insert storm and preserves the seven-day dedup window", async () => { + const { db, raw } = createInMemoryAdeDb(); + const service = createIngressService(db); + + try { + for (let index = 0; index < 10_000; index += 1) { + await service.dispatchIngressTrigger({ + source: "github-relay", + eventKey: `storm-${index}`, + triggerType: "github.issue_opened", + eventName: "github.issue_opened", + rawPayload: { index, body: "payload that must never reach disk" }, + }); + } + + const count = Number(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where project_id = 'proj'", + ))[0]?.count ?? 0); + expect(count).toBe(2_000); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where raw_payload_json is not null", + ))[0]?.count).toBe(0); + + const original = await service.dispatchIngressTrigger({ + source: "github-relay", + eventKey: "storm-9999", + triggerType: "github.issue_opened", + rawPayload: { replay: "within-window" }, + }); + expect(original).not.toBeNull(); + const storedOriginal = mapExecRows(raw.exec( + "select id from automation_ingress_events where event_key = 'storm-9999'", + )); + expect(storedOriginal).toHaveLength(1); + expect(original?.id).toBe(storedOriginal[0]?.id); + + raw.run( + "update automation_ingress_events set received_at = ? where event_key = 'storm-9999'", + [new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000).toISOString()], + ); + await service.dispatchIngressTrigger({ + source: "github-relay", + eventKey: "ttl-prune-trigger", + triggerType: "github.issue_opened", + }); + const replayed = await service.dispatchIngressTrigger({ + source: "github-relay", + eventKey: "storm-9999", + triggerType: "github.issue_opened", + rawPayload: { replay: "after-window" }, + }); + expect(replayed?.id).not.toBe(original?.id); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where raw_payload_json is not null", + ))[0]?.count).toBe(0); + } finally { + service.dispose(); + } + }, 120_000); + + it("reclaims legacy payloads and local caches once, in bounded chunks", () => { + const { db, raw } = createInMemoryAdeDb(); + raw.run("create table review_run_artifacts(id text primary key, created_at text not null)"); + raw.run("create table pull_request_snapshots(pr_id text primary key, updated_at text not null)"); + + const recent = new Date().toISOString(); + const oldIngress = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000).toISOString(); + const oldReview = new Date(Date.now() - 31 * 24 * 60 * 60 * 1_000).toISOString(); + const oldSnapshot = new Date(Date.now() - 61 * 24 * 60 * 60 * 1_000).toISOString(); + raw.run("begin"); + const insert = raw.prepare(` + insert into automation_ingress_events( + id, project_id, source, event_key, automation_ids_json, trigger_type, + status, raw_payload_json, received_at + ) values (?, 'proj', 'github-relay', ?, '[]', 'github.issue_opened', ?, ?, ?) + `); + for (let index = 0; index < 2_205; index += 1) { + insert.run([ + `legacy-${index}`, + `legacy-key-${index}`, + index < 2_202 ? "ignored" : "dispatched", + JSON.stringify({ index, body: "legacy payload" }), + index < 100 ? oldIngress : recent, + ]); + } + insert.free(); + raw.run("commit"); + raw.run("insert into review_run_artifacts values ('old-review', ?), ('new-review', ?)", [oldReview, recent]); + raw.run("insert into pull_request_snapshots values ('old-pr', ?), ('new-pr', ?)", [oldSnapshot, recent]); + + const info = vi.fn(); + const logger = { ...createLogger(), info } as any; + const first = createIngressService(db, logger); + try { + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where raw_payload_json is not null", + ))[0]?.count).toBe(0); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where project_id = 'proj'", + ))[0]?.count).toBe(2_000); + expect(mapExecRows(raw.exec("select id from review_run_artifacts order by id"))).toEqual([{ id: "new-review" }]); + expect(mapExecRows(raw.exec("select pr_id from pull_request_snapshots order by pr_id"))).toEqual([{ pr_id: "new-pr" }]); + expect(info).toHaveBeenCalledWith("automations.ingress_payload_reclaim", { + rowsCleared: 2_205, + bytesCleared: expect.any(Number), + }); + expect(Number(info.mock.calls[0]?.[1]?.bytesCleared ?? 0)).toBeGreaterThan(0); + + const logCount = info.mock.calls.length; + const second = createIngressService(db, logger); + second.dispose(); + expect(info).toHaveBeenCalledTimes(logCount); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where project_id = 'proj'", + ))[0]?.count).toBe(2_000); + } finally { + first.dispose(); + } + }); +}); diff --git a/apps/desktop/src/main/services/automations/automationService.ts b/apps/desktop/src/main/services/automations/automationService.ts index f27556af6..16c0fa595 100644 --- a/apps/desktop/src/main/services/automations/automationService.ts +++ b/apps/desktop/src/main/services/automations/automationService.ts @@ -354,10 +354,15 @@ type AutomationIngressEventRow = { summary: string | null; error_message: string | null; cursor: string | null; - raw_payload_json: string | null; received_at: string; }; +const INGRESS_EVENT_RETENTION_MS = 7 * 24 * 60 * 60 * 1_000; +const INGRESS_EVENT_MAX_ROWS_PER_PROJECT = 2_000; +const INGRESS_PAYLOAD_RECLAIM_CHUNK_ROWS = 2_000; +const REVIEW_ARTIFACT_RETENTION_MS = 30 * 24 * 60 * 60 * 1_000; +const PR_SNAPSHOT_RETENTION_MS = 60 * 24 * 60 * 60 * 1_000; + type AutomationPendingPublishRow = { id: string; run_id: string; @@ -1121,6 +1126,93 @@ export function createAutomationService({ } }; + const tableExists = (tableName: string): boolean => Boolean(db.get( + "select 1 as present from sqlite_master where type = 'table' and name = ? limit 1", + [tableName], + )); + + const pruneIngressEventsForProject = (targetProjectId: string, referenceTime = new Date()): void => { + const cutoff = new Date(referenceTime.getTime() - INGRESS_EVENT_RETENTION_MS).toISOString(); + db.run( + "delete from automation_ingress_events where project_id = ? and received_at < ?", + [targetProjectId, cutoff], + ); + db.run( + `delete from automation_ingress_events + where rowid in ( + select rowid + from automation_ingress_events + where project_id = ? + order by received_at desc, rowid desc + limit -1 offset ${INGRESS_EVENT_MAX_ROWS_PER_PROJECT} + )`, + [targetProjectId], + ); + }; + + const reclaimLegacyIngressPayloads = (): void => { + const hasPayloads = db.get<{ present: number }>( + "select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1", + ); + if (!hasPayloads) return; + + const payloadStats = db.get<{ rows_cleared: number; bytes_cleared: number | null }>( + `select count(*) as rows_cleared, coalesce(sum(length(raw_payload_json)), 0) as bytes_cleared + from automation_ingress_events + where raw_payload_json is not null`, + ); + + while (db.get("select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1")) { + db.run("begin immediate"); + try { + db.run( + `update automation_ingress_events + set raw_payload_json = null + where rowid in ( + select rowid + from automation_ingress_events + where raw_payload_json is not null + order by rowid + limit ${INGRESS_PAYLOAD_RECLAIM_CHUNK_ROWS} + )`, + ); + db.run("commit"); + } catch (error) { + try { + db.run("rollback"); + } catch { + // Preserve the original reclaim failure. + } + throw error; + } + } + + const referenceTime = new Date(); + for (const row of db.all<{ project_id: string }>( + "select distinct project_id from automation_ingress_events", + )) { + pruneIngressEventsForProject(row.project_id, referenceTime); + } + + if (tableExists("review_run_artifacts")) { + db.run( + "delete from review_run_artifacts where created_at < ?", + [new Date(referenceTime.getTime() - REVIEW_ARTIFACT_RETENTION_MS).toISOString()], + ); + } + if (tableExists("pull_request_snapshots")) { + db.run( + "delete from pull_request_snapshots where updated_at < ?", + [new Date(referenceTime.getTime() - PR_SNAPSHOT_RETENTION_MS).toISOString()], + ); + } + + logger.info("automations.ingress_payload_reclaim", { + rowsCleared: Number(payloadStats?.rows_cleared ?? 0), + bytesCleared: Number(payloadStats?.bytes_cleared ?? 0), + }); + }; + const ensureSchema = () => { const runColumns = [ ["chat_session_id", "alter table automation_runs add column chat_session_id text"], @@ -1271,6 +1363,8 @@ export function createAutomationService({ // delete completed before the crash, the missing-lane path records success. db.run("update automation_scheduled_cleanups set status = 'scheduled' where status = 'executing' and project_id = ?", [projectId]); + reclaimLegacyIngressPayloads(); + }; ensureSchema(); @@ -1959,7 +2053,7 @@ export function createAutomationService({ ` select id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, - cursor, raw_payload_json, received_at + cursor, received_at from automation_ingress_events where project_id = ? and id = ? limit 1 @@ -1983,7 +2077,7 @@ export function createAutomationService({ ` select id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, - cursor, raw_payload_json, received_at + cursor, received_at from automation_ingress_events where project_id = ? order by received_at desc @@ -3284,7 +3378,7 @@ export function createAutomationService({ ` select id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, - cursor, raw_payload_json, received_at + cursor, received_at from automation_ingress_events where project_id = ? and source = ? and event_key = ? limit 1 @@ -3295,25 +3389,36 @@ export function createAutomationService({ const eventId = randomUUID(); const receivedAt = nowIso(); - db.run( - ` - insert into automation_ingress_events( - id, project_id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, cursor, raw_payload_json, received_at - ) values (?, ?, ?, ?, '[]', ?, ?, 'received', ?, null, ?, ?, ?) - `, - [ - eventId, - projectId, - args.source, - eventKey, - args.triggerType, - args.eventName ?? null, - args.summary ?? null, - args.cursor ?? null, - args.rawPayload ? JSON.stringify(args.rawPayload) : null, - receivedAt, - ] - ); + db.run("begin immediate"); + try { + db.run( + ` + insert into automation_ingress_events( + id, project_id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, cursor, raw_payload_json, received_at + ) values (?, ?, ?, ?, '[]', ?, ?, 'received', ?, null, ?, null, ?) + `, + [ + eventId, + projectId, + args.source, + eventKey, + args.triggerType, + args.eventName ?? null, + args.summary ?? null, + args.cursor ?? null, + receivedAt, + ] + ); + pruneIngressEventsForProject(projectId, new Date(receivedAt)); + db.run("commit"); + } catch (error) { + try { + db.run("rollback"); + } catch { + // Preserve the original insert/prune failure. + } + throw error; + } if (args.cursor) upsertIngressCursor(args.source, args.cursor); const trigger: TriggerContext = { diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 5db779d9d..212258154 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -641,6 +641,8 @@ import type { createLinearIssueTracker } from "../cto/linearIssueTracker"; import type { createUsageTrackingService } from "../usage/usageTrackingService"; import type { createStorageInsightsService } from "../storage/storageInsightsService"; import type { + MaintenanceRunReport, + RuntimeHealthSnapshot, StorageCleanupPreview, StorageCleanupResult, StorageCleanupTarget, @@ -3730,6 +3732,16 @@ export function registerIpc({ ); }); + // Machine-level daemon health (slow-action window). Direct IPC like + // getResourceUsage: the connection pool lives in Electron main, so there is no + // action-domain routing and no runtime-backed null-service risk. + ipcMain.handle(IPC.appGetRuntimeHealth, async (): Promise => { + return ( + localRuntimeConnectionPool?.getRuntimeHealth?.() + ?? { slowActions24h: 0, slowActionP95Ms: null, sampledAt: new Date().toISOString() } + ); + }); + ipcMain.handle(IPC.storageGetPressure, async (): Promise => { const monitor = requireAppContextValue(getCtx(), "diskPressureMonitor"); return monitor.getSnapshot({ maxAgeMs: 1_000 }); @@ -3748,6 +3760,11 @@ export function registerIpc({ return service.compressNow(); }); + ipcMain.handle(IPC.storageRunMaintenanceNow, async (): Promise => { + const service = requireAppContextValue(getCtx(), "storageInsightsService"); + return service.runMaintenanceNow(); + }); + ipcMain.handle(IPC.storageCleanupPreview, async ( _event, targets: StorageCleanupTarget[], diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts index f4e0fae00..24df05c4d 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts @@ -172,6 +172,35 @@ async function shutdownRuntime(socketPath: string): Promise { } describe("local runtime connection pool", () => { + it("aggregates slow actions into a bounded 24h runtime-health window", () => { + const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const pool = new LocalRuntimeConnectionPool("1.2.3", logger as never) as unknown as { + recordSlowAction: (atMs: number, totalMs: number) => void; + getRuntimeHealth: (nowMs?: number) => { + slowActions24h: number; + slowActionP95Ms: number | null; + sampledAt: string; + }; + dispose: () => void; + }; + const now = Date.parse("2026-07-17T12:00:00.000Z"); + + // An empty window reports zero slow actions and a null p95. + expect(pool.getRuntimeHealth(now)).toMatchObject({ slowActions24h: 0, slowActionP95Ms: null }); + + // One sample 25h old (out of window) plus 100 in-window samples 501..600 ms. + pool.recordSlowAction(now - 25 * 60 * 60_000, 9_000); + for (let index = 0; index < 100; index += 1) { + pool.recordSlowAction(now - index * 60_000, 501 + index); + } + const health = pool.getRuntimeHealth(now); + expect(health.slowActions24h).toBe(100); // the 25h-old sample is pruned + // Nearest-rank p95 of 501..600 → index ceil(0.95*100)-1 = 94 → 595. + expect(health.slowActionP95Ms).toBe(595); + expect(health.sampledAt).toBe(new Date(now).toISOString()); + pool.dispose(); + }); + it("compares ADE runtime versions without requiring exact tag formatting", () => { expect(compareRuntimeVersionStrings("v1.2.14", "1.2.13")).toBe(1); expect(compareRuntimeVersionStrings("1.2.12", "v1.2.13")).toBe(-1); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts index cfa0e66fd..b408b2132 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts @@ -38,6 +38,13 @@ import { buildPackagedRuntimeNodePath, type PackagedRuntimeNodePathOptions } fro import { readLastFailure } from "../runtime/lastFailureStore"; import type { AdeRecoveryErrorCode } from "../../../shared/types/recovery"; import { LOCAL_RELEASE_BUILD_OUTPUT_RUNTIME_MESSAGE } from "../../../shared/runtimeErrors"; +import type { RuntimeHealthSnapshot } from "../../../shared/types/storage"; + +const SLOW_ACTION_THRESHOLD_MS = 500; +const RUNTIME_HEALTH_WINDOW_MS = 24 * 60 * 60_000; +// Ring-cap the in-memory slow-action window so a sustained slow-call storm can +// never grow this array without bound (the very failure mode we are surfacing). +const RUNTIME_HEALTH_MAX_SAMPLES = 5_000; type LocalRuntimeConnection = { client: RuntimeRpcClient; @@ -45,6 +52,8 @@ type LocalRuntimeConnection = { socketPath: string; }; +type SlowActionSample = { at: number; totalMs: number }; + type RuntimeEventNotification = { subscriptionId: string; projectId: string; @@ -685,6 +694,9 @@ export class LocalRuntimeConnectionPool { }; private serviceHealthCheckedAtMs = 0; private serviceInstallPromise: Promise | null = null; + // Rolling 24 h aggregate of slow (>500 ms) or errored daemon action calls. + // Feeds the machine-level runtime-health diagnostic surfaced in Settings. + private slowActionSamples: SlowActionSample[] = []; constructor( private readonly appVersion: string, @@ -722,6 +734,37 @@ export class LocalRuntimeConnectionPool { return Array.from(new Set(pids)); } + private recordSlowAction(atMs: number, totalMs: number): void { + this.slowActionSamples.push({ at: atMs, totalMs }); + // Keep the window bounded on both age and count. + const horizon = atMs - RUNTIME_HEALTH_WINDOW_MS; + if (this.slowActionSamples.length > RUNTIME_HEALTH_MAX_SAMPLES || this.slowActionSamples[0]!.at < horizon) { + this.slowActionSamples = this.slowActionSamples.filter((sample) => sample.at >= horizon); + if (this.slowActionSamples.length > RUNTIME_HEALTH_MAX_SAMPLES) { + this.slowActionSamples = this.slowActionSamples.slice(-RUNTIME_HEALTH_MAX_SAMPLES); + } + } + } + + getRuntimeHealth(nowMs: number = Date.now()): RuntimeHealthSnapshot { + const horizon = nowMs - RUNTIME_HEALTH_WINDOW_MS; + const recent = this.slowActionSamples.filter((sample) => sample.at >= horizon); + // Prune while we are here so idle pools do not retain a stale window. + this.slowActionSamples = recent; + let p95: number | null = null; + if (recent.length > 0) { + const sorted = recent.map((sample) => sample.totalMs).sort((left, right) => left - right); + // Nearest-rank p95: index ceil(0.95*n)-1, clamped into range. + const rank = Math.min(sorted.length - 1, Math.max(0, Math.ceil(0.95 * sorted.length) - 1)); + p95 = sorted[rank]!; + } + return { + slowActions24h: recent.length, + slowActionP95Ms: p95, + sampledAt: new Date(nowMs).toISOString(), + }; + } + noteServiceInstallSkipped(message: string): void { this.serviceInstallStatus = { state: "skipped", @@ -1193,7 +1236,8 @@ export class LocalRuntimeConnectionPool { } finally { const tCall = Date.now(); const totalMs = tCall - tStart; - if (totalMs > 500 || callError) { + if (totalMs > SLOW_ACTION_THRESHOLD_MS || callError) { + this.recordSlowAction(tCall, totalMs); this.logger.warn("local_runtime.action_slow", { domain: request.domain, action: request.action, diff --git a/apps/desktop/src/main/services/state/dbMaintenanceApi.ts b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts new file mode 100644 index 000000000..16d4074e9 --- /dev/null +++ b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts @@ -0,0 +1,35 @@ +// Interface for DB retention/maintenance hooks the storage doctor invokes. +// Implemented by kvDb (attached to the KvDb handle as `maintenance`); consumed +// by storageInsightsService via optional chaining so the doctor degrades +// gracefully on handles that predate the implementation. + +export type DbMaintenanceResult = { + itemsAffected: number; + bytesReclaimed: number; + skippedReason?: "has_peers" | "below_threshold" | "unsupported" | null; +}; + +export interface DbMaintenanceApi { + /** Age+count prune of automation_ingress_events (7d / 2,000 per project). */ + pruneIngressEvents(): DbMaintenanceResult; + /** Delete review_run_artifacts rows older than 30 days. */ + pruneReviewArtifacts(): DbMaintenanceResult; + /** Delete pull_request_snapshots rows not updated in 60 days. */ + prunePrSnapshots(): DbMaintenanceResult; + /** + * Reclaim cr-sqlite clock/pks bookkeeping. Only safe (and only performed) + * when the project has zero sync peers; otherwise returns skippedReason + * "has_peers" without touching anything. + */ + compactCrsqlTombstones(): DbMaintenanceResult; + /** + * Full VACUUM (+ auto_vacuum=INCREMENTAL activation) when the freelist + * fraction exceeds `threshold`; bounded incremental_vacuum chunks otherwise. + * Returns bytes reclaimed on disk. + */ + vacuumIfFragmented(threshold: number): DbMaintenanceResult; +} + +export interface KvDbWithMaintenance { + maintenance?: DbMaintenanceApi; +} diff --git a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts index b063466b7..d0f64f5ef 100644 --- a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts +++ b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts @@ -8,6 +8,7 @@ import { constrainSqliteMaxPages } from "../../../test/faultInjection"; import { classifySqliteOpenError, openKvDb, + openReadonlyDatabase, rebuildTableInTransaction, recoverInterruptedTableRebuilds, type TableRebuildPlan, @@ -386,3 +387,137 @@ describe("kvDb migration backup", () => { expect(classifySqliteOpenError(new Error("surprise"))).toBe("unknown"); }); }); + +describe("kvDb storage maintenance", () => { + it("switches a delete-journal database to WAL with NORMAL synchronous writes", async () => { + const dbPath = makeDbPath(); + const raw = new DatabaseSync(dbPath); + raw.exec("pragma journal_mode = delete; create table seed(id integer primary key, value text)"); + raw.close(); + + const readonly = openReadonlyDatabase(dbPath); + expect((readonly.prepare("pragma journal_mode").get() as { journal_mode: string }).journal_mode).toBe("delete"); + readonly.close(); + + const db = await openKvDb(dbPath, createLogger()); + closeLater(db); + expect(db.get<{ journal_mode: string }>("pragma journal_mode")?.journal_mode).toBe("wal"); + expect(db.get<{ synchronous: number }>("pragma synchronous")?.synchronous).toBe(1); + }); + + it("reclaims a fragmented file, activates incremental auto-vacuum, and skips a healthy file", async () => { + const dbPath = makeDbPath(); + const db = await openKvDb(dbPath, createLogger()); + closeLater(db); + db.run("create table maintenance_fragmentation(id integer primary key, payload text not null)"); + db.run("begin immediate"); + for (let index = 0; index < 2_000; index += 1) { + db.run( + "insert into maintenance_fragmentation(id, payload) values (?, ?)", + [index, `${index}-${"x".repeat(4_000)}`], + ); + } + db.run("commit"); + db.flushNow(); + db.run("delete from maintenance_fragmentation"); + db.flushNow(); + + const beforeBytes = fs.statSync(dbPath).size; + const pageCount = Number(db.get<{ page_count: number }>("pragma page_count")?.page_count ?? 0); + const freePages = Number(db.get<{ freelist_count: number }>("pragma freelist_count")?.freelist_count ?? 0); + expect(freePages / pageCount).toBeGreaterThan(0.2); + + const result = db.maintenance?.vacuumIfFragmented(0.2); + expect(result?.skippedReason).toBeNull(); + expect(result?.itemsAffected).toBeGreaterThan(0); + expect(result?.bytesReclaimed).toBeGreaterThan(0); + expect(fs.statSync(dbPath).size).toBeLessThan(beforeBytes); + expect(db.get<{ auto_vacuum: number }>("pragma auto_vacuum")?.auto_vacuum).toBe(2); + + db.run("begin immediate"); + for (let index = 0; index < 256; index += 1) { + db.run( + "insert into maintenance_fragmentation(id, payload) values (?, ?)", + [index, `${index}-${"y".repeat(4_000)}`], + ); + } + db.run("commit"); + db.run("delete from maintenance_fragmentation"); + const incrementalFreePages = Number(db.get<{ freelist_count: number }>("pragma freelist_count")?.freelist_count ?? 0); + expect(incrementalFreePages).toBeGreaterThan(0); + const incrementalResult = db.maintenance?.vacuumIfFragmented(1); + expect(incrementalResult?.skippedReason).toBeNull(); + expect(incrementalResult?.itemsAffected).toBeGreaterThan(0); + + const healthyPath = makeDbPath(); + const healthy = await openKvDb(healthyPath, createLogger()); + closeLater(healthy); + const healthyResult = healthy.maintenance?.vacuumIfFragmented(0.9); + expect(healthyResult).toMatchObject({ + itemsAffected: 0, + skippedReason: "below_threshold", + }); + expect(healthy.get<{ auto_vacuum: number }>("pragma auto_vacuum")?.auto_vacuum).toBe(0); + expect(healthy.maintenance?.vacuumIfFragmented(Number.NaN)).toEqual({ + itemsAffected: 0, + bytesReclaimed: 0, + skippedReason: "unsupported", + }); + }); + + it("gates CRR tombstone compaction on peer state", async () => { + const pairedPath = makeDbPath(); + const paired = await openKvDb(pairedPath, createLogger(), { hasSyncPeers: () => true }); + closeLater(paired); + expect(paired.maintenance?.compactCrsqlTombstones()).toEqual({ + itemsAffected: 0, + bytesReclaimed: 0, + skippedReason: "has_peers", + }); + }); + + it.skipIf(process.platform === "linux")("compacts CRR tombstones with no peers and preserves operations byte-for-byte", async () => { + const dbPath = makeDbPath(); + const db = await openKvDb(dbPath, createLogger(), { hasSyncPeers: () => false }); + closeLater(db); + expect(db.sync.isAvailable?.(), "cr-sqlite must be available for the compaction contract test").toBe(true); + + const now = "2026-07-17T12:00:00.000Z"; + db.run( + `insert into projects(id, root_path, display_name, default_base_ref, created_at, last_opened_at) + values (?, ?, ?, ?, ?, ?)`, + ["project-maintenance", "/repo/maintenance", "Maintenance", "main", now, now], + ); + db.run("begin immediate"); + for (let index = 0; index < 250; index += 1) { + db.run( + `insert into operations( + id, project_id, lane_id, kind, started_at, ended_at, status, + pre_head_sha, post_head_sha, metadata_json + ) values (?, 'project-maintenance', null, 'test', ?, ?, 'succeeded', null, null, ?)`, + [`operation-${index}`, now, now, JSON.stringify({ index, payload: "x".repeat(1_000) })], + ); + } + db.run("commit"); + db.run("delete from operations where id not in (select id from operations order by id desc limit 10)"); + + const before = db.all>("select * from operations order by id"); + const shadowRowsBefore = Number(db.get<{ count: number }>( + `select + (select count(*) from operations__crsql_pks) + + (select count(*) from operations__crsql_clock) as count`, + )?.count ?? 0); + expect(shadowRowsBefore).toBeGreaterThan(before.length); + + const result = db.maintenance?.compactCrsqlTombstones(); + expect(result?.skippedReason).toBeNull(); + expect(result?.itemsAffected).toBeGreaterThan(0); + expect(db.all>("select * from operations order by id")).toEqual(before); + const shadowRowsAfter = Number(db.get<{ count: number }>( + `select + (select count(*) from operations__crsql_pks) + + (select count(*) from operations__crsql_clock) as count`, + )?.count ?? 0); + expect(shadowRowsAfter).toBeLessThan(shadowRowsBefore); + }); +}); diff --git a/apps/desktop/src/main/services/state/kvDb.ts b/apps/desktop/src/main/services/state/kvDb.ts index 924f695ac..f356b61b5 100644 --- a/apps/desktop/src/main/services/state/kvDb.ts +++ b/apps/desktop/src/main/services/state/kvDb.ts @@ -9,6 +9,7 @@ import type { Logger } from "../logging/logger"; import { safeJsonParse } from "../shared/utils"; import { isNoSpaceError, readVolumeSpace } from "../storage/volume"; import { resolveCrsqliteExtensionPath } from "./crsqliteExtension"; +import type { DbMaintenanceApi, DbMaintenanceResult } from "./dbMaintenanceApi"; import type { ApplyRemoteChangesResult, CrsqlChangeRow, SyncScalar } from "../../../shared/types/sync"; type DatabaseSyncConstructor = new (dbPath: string, options?: { allowExtension?: boolean; readOnly?: boolean }) => DatabaseSyncType; @@ -121,6 +122,9 @@ export type AdeDb = { sync: AdeDbSyncApi; + /** Retention and compaction hooks consumed by the storage doctor. */ + maintenance?: DbMaintenanceApi; + /** * Force pending WAL frames onto the main database before shutdown. This uses a * TRUNCATE checkpoint and can wait for active readers, so keep calls on @@ -151,6 +155,8 @@ function openRawDatabase(dbPath: string): DatabaseSyncType { // Allow concurrent access from multiple ADE processes (e.g. dogfooding). // Without this, a second instance gets SQLITE_BUSY immediately on writes. db.exec("PRAGMA busy_timeout = 5000"); + db.exec("PRAGMA journal_mode = WAL"); + db.exec("PRAGMA synchronous = NORMAL"); return db; } @@ -3484,7 +3490,28 @@ function loadCrsqlite(db: DatabaseSyncType, extensionPath: string): void { db.loadExtension(extensionPath); } -export async function openKvDb(dbPath: string, logger: Logger): Promise { +export type OpenKvDbOptions = { + /** Conservative by default: compaction is unsafe unless zero sync peers is proven. */ + hasSyncPeers?: () => boolean; +}; + +function sqliteFootprintBytes(dbPath: string): number { + let total = 0; + for (const filePath of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) { + try { + total += fs.statSync(filePath).size; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + return total; +} + +export async function openKvDb( + dbPath: string, + logger: Logger, + options: OpenKvDbOptions = {}, +): Promise { const extensionPath = resolveCrsqliteExtensionPath(); const hasCrsqlite = extensionPath != null; const desiredSiteId = ensureLocalSiteIdFile(dbPath); @@ -3632,6 +3659,167 @@ export async function openKvDb(dbPath: string, logger: Logger): Promise { const all = crrAwareDb.all; const get = crrAwareDb.get; + const unsupportedMaintenanceResult = (): DbMaintenanceResult => ({ + itemsAffected: 0, + bytesReclaimed: 0, + skippedReason: "unsupported", + }); + + const runMaintenanceSafely = ( + action: keyof DbMaintenanceApi, + operation: () => DbMaintenanceResult, + ): DbMaintenanceResult => { + try { + return operation(); + } catch (error) { + logger.warn("db.maintenance_failed", { + action, + error: error instanceof Error ? error.message : String(error), + }); + return unsupportedMaintenanceResult(); + } + }; + + const vacuumIfFragmented = (threshold: number): DbMaintenanceResult => { + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + throw new Error(`Invalid fragmentation threshold: ${String(threshold)}`); + } + + const beforeBytes = sqliteFootprintBytes(dbPath); + const pageCount = Number(getRow<{ page_count: number }>(db, "pragma page_count")?.page_count ?? 0); + const freePagesBefore = Number(getRow<{ freelist_count: number }>(db, "pragma freelist_count")?.freelist_count ?? 0); + const fragmentation = pageCount > 0 ? freePagesBefore / pageCount : 0; + const autoVacuum = Number(getRow<{ auto_vacuum: number }>(db, "pragma auto_vacuum")?.auto_vacuum ?? 0); + let itemsAffected = 0; + let skippedReason: DbMaintenanceResult["skippedReason"] = null; + let mode: "full" | "incremental" | "none" = "none"; + + if (pageCount > 0 && fragmentation >= threshold) { + // SQLite applies a NONE -> INCREMENTAL auto-vacuum transition when the + // following VACUUM rebuilds the file, so the pragma must come first. + db.exec("PRAGMA auto_vacuum = INCREMENTAL"); + db.exec("VACUUM"); + itemsAffected = freePagesBefore; + mode = "full"; + } else if (autoVacuum === 2 && freePagesBefore > 0) { + db.exec("PRAGMA incremental_vacuum(2000)"); + const freePagesAfter = Number(getRow<{ freelist_count: number }>(db, "pragma freelist_count")?.freelist_count ?? 0); + itemsAffected = Math.max(0, freePagesBefore - freePagesAfter); + mode = "incremental"; + } else { + skippedReason = "below_threshold"; + } + + getRow(db, "pragma wal_checkpoint(TRUNCATE)"); + const afterBytes = sqliteFootprintBytes(dbPath); + const bytesReclaimed = Math.max(0, beforeBytes - afterBytes); + logger.info("db.maintenance_vacuum", { + mode, + fragmentation, + threshold, + freePagesBefore, + itemsAffected, + bytesReclaimed, + }); + return { itemsAffected, bytesReclaimed, skippedReason }; + }; + + const maintenance: DbMaintenanceApi = { + pruneIngressEvents: () => runMaintenanceSafely("pruneIngressEvents", () => { + if (!rawHasTable(db, "automation_ingress_events")) return unsupportedMaintenanceResult(); + const cutoff = new Date(Date.now() - 7 * 24 * 60 * 60 * 1_000).toISOString(); + let itemsAffected = runStatement( + db, + "delete from automation_ingress_events where received_at < ?", + [cutoff], + ).changes; + const projects = allRows<{ project_id: string }>( + db, + "select distinct project_id from automation_ingress_events", + ); + for (const project of projects) { + itemsAffected += runStatement( + db, + `delete from automation_ingress_events + where rowid in ( + select rowid + from automation_ingress_events + where project_id = ? + order by received_at desc, rowid desc + limit -1 offset 2000 + )`, + [project.project_id], + ).changes; + } + return { itemsAffected, bytesReclaimed: 0, skippedReason: null }; + }), + pruneReviewArtifacts: () => runMaintenanceSafely("pruneReviewArtifacts", () => { + if (!rawHasTable(db, "review_run_artifacts")) return unsupportedMaintenanceResult(); + const cutoff = new Date(Date.now() - 30 * 24 * 60 * 60 * 1_000).toISOString(); + const itemsAffected = runStatement( + db, + "delete from review_run_artifacts where created_at < ?", + [cutoff], + ).changes; + return { itemsAffected, bytesReclaimed: 0, skippedReason: null }; + }), + prunePrSnapshots: () => runMaintenanceSafely("prunePrSnapshots", () => { + if (!rawHasTable(db, "pull_request_snapshots")) return unsupportedMaintenanceResult(); + const cutoff = new Date(Date.now() - 60 * 24 * 60 * 60 * 1_000).toISOString(); + const itemsAffected = runStatement( + db, + "delete from pull_request_snapshots where updated_at < ?", + [cutoff], + ).changes; + return { itemsAffected, bytesReclaimed: 0, skippedReason: null }; + }), + compactCrsqlTombstones: () => runMaintenanceSafely("compactCrsqlTombstones", () => { + const hasSyncPeers = options.hasSyncPeers ?? (() => true); + if (hasSyncPeers()) { + return { itemsAffected: 0, bytesReclaimed: 0, skippedReason: "has_peers" }; + } + if ( + !crsqliteLoaded + || !rawHasTable(db, "operations") + || !rawHasTable(db, "operations__crsql_pks") + || !rawHasTable(db, "operations__crsql_clock") + ) { + return unsupportedMaintenanceResult(); + } + + const shadowRowsBefore = Number(getRow<{ count: number }>( + db, + `select + (select count(*) from operations__crsql_pks) + + (select count(*) from operations__crsql_clock) as count`, + )?.count ?? 0); + rebuildCrrTableWithBackfill(db, "operations"); + const shadowRowsAfter = Number(getRow<{ count: number }>( + db, + `select + (select count(*) from operations__crsql_pks) + + (select count(*) from operations__crsql_clock) as count`, + )?.count ?? 0); + const vacuumResult = vacuumIfFragmented(0); + const itemsAffected = Math.max(0, shadowRowsBefore - shadowRowsAfter); + logger.info("db.maintenance_crsql_compacted", { + tableName: "operations", + shadowRowsBefore, + shadowRowsAfter, + bytesReclaimed: vacuumResult.bytesReclaimed, + }); + return { + itemsAffected, + bytesReclaimed: vacuumResult.bytesReclaimed, + skippedReason: null, + }; + }), + vacuumIfFragmented: (threshold) => runMaintenanceSafely( + "vacuumIfFragmented", + () => vacuumIfFragmented(threshold), + ), + }; + const sync: AdeDbSyncApi = { isAvailable: () => crsqliteLoaded, getSiteId: () => desiredSiteId, @@ -3866,6 +4054,7 @@ export async function openKvDb(dbPath: string, logger: Logger): Promise { all, get, sync, + maintenance, flushNow: () => { getRow(db, "pragma wal_checkpoint(TRUNCATE)"); }, diff --git a/apps/desktop/src/main/services/storage/historyCompression.ts b/apps/desktop/src/main/services/storage/historyCompression.ts index 4028daf03..56dcad366 100644 --- a/apps/desktop/src/main/services/storage/historyCompression.ts +++ b/apps/desktop/src/main/services/storage/historyCompression.ts @@ -10,7 +10,11 @@ import type { DiskPressureMonitor } from "./diskPressure"; import { readVolumeSpace } from "./volume"; const DAY_MS = 24 * 60 * 60_000; -const DEFAULT_MIN_AGE_DAYS = 30; +// Inactive history older than this is compressed by the storage doctor. Lowered +// from 30d to 14d: on the reference machine 62 files were >14d (280 MB) that the +// 30d threshold left uncompressed. Compression is transparent (read-back +// decompresses), so a shorter threshold reclaims more without user impact. +const DEFAULT_MIN_AGE_DAYS = 14; export const COMPRESSION_MIN_AGE_MS = DEFAULT_MIN_AGE_DAYS * DAY_MS; const DEFAULT_MAX_FILES = 25; const BETWEEN_FILES_DELAY_MS = 250; diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 90a44b8bd..10b9ff307 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -2,9 +2,14 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { StorageCleanupPreview, StorageCleanupTarget } from "../../../shared/types/storage"; +import type { MaintenanceRunReport, StorageCleanupPreview, StorageCleanupTarget } from "../../../shared/types/storage"; import { openKvDb, type AdeDb } from "../state/kvDb"; -import { createStorageInsightsService } from "./storageInsightsService"; +import { + classifyDbTable, + createStorageInsightsService, + deriveSyncBookkeepingAction, + mapDbBreakdown, +} from "./storageInsightsService"; import { recordLastFailure } from "../runtime/lastFailureStore"; const logger = { @@ -401,4 +406,237 @@ describe("storageInsightsService", () => { expect(freshResult.removed).toEqual([{ path: staging, bytes: 13 }]); expect(fs.existsSync(staging)).toBe(false); }); + + const normalDiskPressure = { + getSnapshot: () => ({ + state: "normal" as const, + freeBytes: 100 * 1024 ** 3, + totalBytes: 200 * 1024 ** 3, + freeFraction: 0.5, + perRoot: [], + sampledAt: new Date().toISOString(), + }), + canPerform: () => ({ allowed: true as const, state: "normal" as const }), + subscribe: () => () => {}, + }; + + it("runs the maintenance doctor: compresses, reaps safe staging/backups/build data, and journals it", async () => { + // Compression candidate (>14d old inactive transcript). + const oldTranscript = path.join(projectRoot, ".ade", "transcripts", "chat", "old.jsonl"); + fs.mkdirSync(path.dirname(oldTranscript), { recursive: true }); + fs.writeFileSync(oldTranscript, `${"repeat history ".repeat(50)}\n`); + const twentyDaysAgo = new Date(Date.now() - 20 * 24 * 60 * 60_000); + fs.utimesSync(oldTranscript, twentyDaysAgo, twentyDaysAgo); + + // .ade/tmp release staging: one stale (reaped), one fresh (kept). + const adeTmpStale = path.join(projectRoot, ".ade", "tmp", "ios-testflight-old"); + const adeTmpFresh = path.join(projectRoot, ".ade", "tmp", "release-fresh"); + writeSized(path.join(adeTmpStale, "ipa.bin"), 40); + writeSized(path.join(adeTmpFresh, "ipa.bin"), 41); + const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60_000); + fs.utimesSync(adeTmpStale, eightDaysAgo, eightDaysAgo); + + // System temp staging in an isolated fixture root (never the real /tmp). + const staging = fs.mkdtempSync(path.join(os.tmpdir(), "ade-storage-doctor-staging-")); + extraPaths.push(staging); + const stageStale = path.join(staging, "ade-old-run"); + writeSized(path.join(stageStale, "artifact.bin"), 50); + fs.utimesSync(stageStale, eightDaysAgo, eightDaysAgo); + + // Obsolete recovery backup (old, db healthy, no recent open failure). + const backup = path.join(projectRoot, ".ade", "ade.db.recovery-doctor.bak"); + writeSized(backup, 60); + fs.utimesSync(backup, eightDaysAgo, eightDaysAgo); + + // iOS DerivedData build cache. + const derived = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); + writeSized(path.join(derived, "module.o"), 70); + + const service = createStorageInsightsService({ + projectRoot, adeHome, db, logger, + diskPressure: normalDiskPressure, + isPathActive: () => false, + stagingTmpDir: staging, + }); + const report = await service.runMaintenanceNow(); + + // Reaping outcomes. + expect(fs.existsSync(adeTmpStale)).toBe(false); + expect(fs.existsSync(adeTmpFresh)).toBe(true); + expect(fs.existsSync(stageStale)).toBe(false); + expect(fs.existsSync(backup)).toBe(false); + expect(fs.existsSync(derived)).toBe(false); + // Compression outcome (transparent gzip). + expect(fs.existsSync(`${oldTranscript}.gz`)).toBe(true); + + // Report correctness. + expect(report.trigger).toBe("manual"); + expect(typeof report.dbSizeBytes).toBe("number"); + const byLedger = Object.fromEntries(report.actions.map((action) => [action.ledgerId, action])); + expect(byLedger["fs.transcripts"]!.itemsAffected).toBe(1); + expect(byLedger["fs.tmp"]!.itemsAffected).toBe(1); + expect(byLedger["fs.tmp_staging"]!.itemsAffected).toBe(1); + expect(byLedger["fs.recovery_backups"]!.itemsAffected).toBe(1); + expect(byLedger["fs.ios_derived_data"]!.itemsAffected).toBe(1); + // DB hooks absent (no maintenance handle) → unsupported skips, not errors. + expect(byLedger["db.automation_ingress_events"]!.skippedReason).toBe("unsupported"); + expect(byLedger["db.automation_ingress_events"]!.error).toBeNull(); + expect(report.reclaimedBytes).toBeGreaterThanOrEqual(40 + 50 + 60 + 70); + + // Journal written to .ade/cache and surfaced through snapshot extras. + const journalPath = path.join(projectRoot, ".ade", "cache", "storage-doctor-journal.json"); + expect(JSON.parse(fs.readFileSync(journalPath, "utf8"))).toHaveLength(1); + const extras = (await service.getSnapshot({ forceRefresh: true })).extras!; + expect(extras.maintenance.lastRun?.trigger).toBe("manual"); + service.dispose(); + }); + + it("isolates a failing maintenance step without aborting the run", async () => { + (db as unknown as { maintenance?: unknown }).maintenance = { + pruneIngressEvents: () => { throw new Error("prune boom"); }, + pruneReviewArtifacts: () => ({ itemsAffected: 2, bytesReclaimed: 128 }), + prunePrSnapshots: () => ({ itemsAffected: 0, bytesReclaimed: 0 }), + compactCrsqlTombstones: () => ({ itemsAffected: 0, bytesReclaimed: 0, skippedReason: "has_peers" }), + vacuumIfFragmented: () => ({ itemsAffected: 0, bytesReclaimed: 512 }), + }; + // Point staging at a nonexistent dir so the reap never scans the real /tmp. + const service = createStorageInsightsService({ + projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), + }); + const report = await service.runMaintenanceNow(); + const byLedger = Object.fromEntries(report.actions.map((action) => [action.ledgerId, action])); + expect(byLedger["db.automation_ingress_events"]!.error).toBe("prune boom"); + expect(byLedger["db.review_run_artifacts"]).toMatchObject({ itemsAffected: 2, bytesReclaimed: 128, error: null }); + expect(byLedger["db.operations_crsql"]!.skippedReason).toBe("has_peers"); + expect(report.reclaimedBytes).toBeGreaterThanOrEqual(128 + 512); + expect(fs.existsSync(path.join(projectRoot, ".ade", "cache", "storage-doctor-journal.json"))).toBe(true); + delete (db as unknown as { maintenance?: unknown }).maintenance; + service.dispose(); + }); + + it("caps the maintenance journal at 30 runs, newest first", async () => { + const cacheDir = path.join(projectRoot, ".ade", "cache"); + fs.mkdirSync(cacheDir, { recursive: true }); + const seed: MaintenanceRunReport[] = Array.from({ length: 30 }, (_value, index) => ({ + startedAt: `2026-01-${String(index + 1).padStart(2, "0")}T00:00:00.000Z`, + finishedAt: `2026-01-${String(index + 1).padStart(2, "0")}T00:00:01.000Z`, + trigger: "daily", + actions: [], + reclaimedBytes: 0, + dbSizeBytes: 1, + })); + fs.writeFileSync(path.join(cacheDir, "storage-doctor-journal.json"), JSON.stringify(seed)); + const service = createStorageInsightsService({ + projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), + }); + const report = await service.runMaintenanceNow(); + const journal = JSON.parse(fs.readFileSync(path.join(cacheDir, "storage-doctor-journal.json"), "utf8")); + expect(journal).toHaveLength(30); + expect(journal[0].trigger).toBe("manual"); + expect(journal[0].startedAt).toBe(report.startedAt); + service.dispose(); + }); + + it("emits one bounded maintenance analytics event per run", async () => { + const captures: Array> = []; + const service = createStorageInsightsService({ + projectRoot, adeHome, db, logger, + projectId: "proj-1", + captureAnalytics: (input) => { captures.push(input as Record); }, + stagingTmpDir: path.join(adeHome, "no-real-staging"), + }); + await service.runMaintenanceNow(); + expect(captures).toHaveLength(1); + expect(captures[0]).toMatchObject({ + event: "ade_feature_used", + surface: "desktop", + projectId: "proj-1", + dedupeKey: "storage_doctor_run:proj-1", + minimumIntervalMs: 20 * 60 * 60_000, + }); + const properties = captures[0]!.properties as Record; + expect(properties).toMatchObject({ feature: "storage_doctor", action: "maintenance_run", outcome: "completed" }); + expect(typeof properties.bytes_freed).toBe("number"); + expect(typeof properties.files_compressed).toBe("number"); + service.dispose(); + }); + + it("populates snapshot extras with db breakdown, policy chips, and safe-reclaimable bytes", async () => { + writeSized(path.join(projectRoot, ".ade", "cache", "browser-observations", "c.bin"), 100); + writeSized(path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData", "m.o"), 200); + const snapshot = await createStorageInsightsService({ + projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), + }).getSnapshot(); + const extras = snapshot.extras!; + expect(Array.isArray(extras.dbBreakdown)).toBe(true); + expect(extras.maintenance.journal).toEqual([]); + expect(extras.maintenance.lastRun).toBeNull(); + expect(extras.policyChips.chats_history).toBe("Compressed after 14 days"); + // safe_to_remove caches (browser-observations 100 + DerivedData 200) are counted. + expect(extras.safeReclaimableBytes).toBeGreaterThanOrEqual(300); + if (extras.dbBreakdown.length > 0) { + expect(extras.dbBreakdown.some((entry) => entry.category === "core")).toBe(true); + } + }); + + it("classifies and aggregates db tables into coarse breakdown categories", () => { + expect(classifyDbTable("automation_ingress_events")).toBe("webhooks"); + expect(classifyDbTable("idx_automation_ingress_events_project_received")).toBe("webhooks"); + expect(classifyDbTable("operations__crsql_clock")).toBe("sync_bookkeeping"); + expect(classifyDbTable("review_run_artifacts")).toBe("review_artifacts"); + expect(classifyDbTable("pull_request_snapshots")).toBe("pr_cache"); + expect(classifyDbTable("chats")).toBe("core"); + + const breakdown = mapDbBreakdown([ + { name: "automation_ingress_events", bytes: 300 }, + { name: "idx_automation_ingress_events_x", bytes: 100 }, + { name: "operations__crsql_clock", bytes: 90 }, + { name: "chats", bytes: 500 }, + { name: "lanes", bytes: 0 }, + ]); + const byCategory = Object.fromEntries(breakdown.map((entry) => [entry.category, entry])); + expect(byCategory.webhooks).toMatchObject({ bytes: 400, label: "Webhook history", action: "prunable" }); + expect(byCategory.sync_bookkeeping!.action).toBe("compactable"); + expect(byCategory.core).toMatchObject({ bytes: 500, action: null }); + // Sorted by bytes desc: core (500) before webhooks (400). + expect(breakdown[0]!.category).toBe("core"); + + // Override replaces only the sync_bookkeeping action label. + const pending = mapDbBreakdown( + [{ name: "operations__crsql_clock", bytes: 90 }, { name: "chats", bytes: 500 }], + { syncBookkeeping: "compaction_pending" }, + ); + expect(pending.find((entry) => entry.category === "sync_bookkeeping")!.action).toBe("compaction_pending"); + expect(pending.find((entry) => entry.category === "core")!.action).toBeNull(); + }); + + it("derives the sync-bookkeeping label from the last journal run's compaction skip", () => { + const run = (compactAction: Record | null): MaintenanceRunReport => ({ + startedAt: "2026-07-17T00:00:00.000Z", + finishedAt: "2026-07-17T00:00:01.000Z", + trigger: "daily", + actions: compactAction ? [compactAction as never] : [], + reclaimedBytes: 0, + dbSizeBytes: 1, + }); + // No journal yet → compactable. + expect(deriveSyncBookkeepingAction([])).toBe("compactable"); + // Last run was peer-blocked → waiting to compact. + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: "has_peers" }), + ])).toBe("compaction_pending"); + // Last run actually compacted (no peer skip) → compactable. + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: null, itemsAffected: 3 }), + ])).toBe("compactable"); + // Hook absent last run (unsupported) → still compactable, never lies "pending". + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: "unsupported" }), + ])).toBe("compactable"); + // Newest run wins even if an older run was peer-blocked. + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: null }), + run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: "has_peers" }), + ])).toBe("compactable"); + }); }); diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index 6bce7ac64..ccd93d39f 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -3,6 +3,11 @@ import os from "node:os"; import path from "node:path"; import { resolveAdeLayout } from "../../../shared/adeLayout"; import type { + DbBreakdownEntry, + MaintenanceAction, + MaintenanceActionKind, + MaintenanceRunReport, + MaintenanceTrigger, StorageCategoryId, StorageCategorySnapshot, StorageCleanupPreview, @@ -12,11 +17,17 @@ import type { StorageItem, StorageSafety, StorageSnapshot, + StorageSnapshotExtras, } from "../../../shared/types/storage"; +import type { + ProductAnalyticsCapture, + ProductAnalyticsCaptureResult, +} from "../../../shared/types/productAnalytics"; import { runGit } from "../git/git"; import type { Logger } from "../logging/logger"; import type { AdeDb } from "../state/kvDb"; import { runQuickCheck } from "../state/kvDb"; +import type { KvDbWithMaintenance } from "../state/dbMaintenanceApi"; import { readLastFailure } from "../runtime/lastFailureStore"; import { isEnoentError } from "../shared/utils"; import type { DiskPressureMonitor } from "./diskPressure"; @@ -26,6 +37,7 @@ import { type CompressionRoots, type CompressionSweepSummary, } from "./historyCompression"; +import { deriveCategoryPolicyChips } from "./storageLedger"; import { readVolumeSpace } from "./volume"; const DEFAULT_CACHE_TTL_MS = 5 * 60_000; @@ -38,6 +50,18 @@ const COMPRESSIBLE_AGE_MS = COMPRESSION_MIN_AGE_MS; const RECOVERY_BACKUP_PATTERN = /(?:\.pre-crsqlite-w1\.bak|\.recovery-.*\.bak)$/; const HISTORY_SWEEP_START_DELAY_MS = 10 * 60_000; const HISTORY_SWEEP_INTERVAL_MS = 24 * 60 * 60_000; +const MAINTENANCE_JOURNAL_FILENAME = "storage-doctor-journal.json"; +const MAINTENANCE_JOURNAL_MAX_RUNS = 30; +// One completed maintenance run per project per 20 h reaches PostHog; the +// storage doctor runs daily, so the dedupe interval collapses repeat runs +// (e.g. daily + a manual "Clean up now") into a single analytics event. +const MAINTENANCE_ANALYTICS_MIN_INTERVAL_MS = 20 * 60 * 60_000; +const VACUUM_FREELIST_THRESHOLD = 0.2; + +/** Injected product-analytics capture. Structurally matches the shared service. */ +export type StorageDoctorAnalyticsCapture = ( + input: ProductAnalyticsCapture, +) => ProductAnalyticsCaptureResult | void; type LaneRow = { id: string; @@ -80,8 +104,83 @@ export type StorageInsightsServiceOptions = { scanBudgetMs?: number; diskPressure?: DiskPressureMonitor | null; isPathActive?: (path: string) => boolean; + /** Salted before send; used only to scope the maintenance analytics dedupe. */ + projectId?: string | null; + /** Emits the per-run `ade_feature_used` maintenance event at the daemon boundary. */ + captureAnalytics?: StorageDoctorAnalyticsCapture | null; + /** + * Root scanned for `ade-*` build/release staging directories. Defaults to + * `os.tmpdir()`; overridable so tests never touch the real system temp dir. + */ + stagingTmpDir?: string; +}; + +type DbBreakdownCategoryKey = DbBreakdownEntry["category"]; + +const DB_BREAKDOWN_META: Record< + DbBreakdownCategoryKey, + { label: string; table: string; action: DbBreakdownEntry["action"] } +> = { + webhooks: { label: "Webhook history", table: "automation_ingress_events", action: "prunable" }, + sync_bookkeeping: { label: "Sync bookkeeping", table: "operations__crsql", action: "compactable" }, + review_artifacts: { label: "Review artifacts", table: "review_run_artifacts", action: "prunable" }, + pr_cache: { label: "PR cache", table: "pull_request_snapshots", action: "prunable" }, + core: { label: "Core data", table: "core", action: null }, }; +/** Classify a dbstat table/index name into a coarse storage-breakdown category. */ +export function classifyDbTable(name: string): DbBreakdownCategoryKey { + const lower = name.toLowerCase(); + if (lower.startsWith("operations__crsql")) return "sync_bookkeeping"; + if (lower.includes("automation_ingress_events")) return "webhooks"; + if (lower.includes("review_run_artifacts")) return "review_artifacts"; + if (lower.includes("pull_request_snapshots")) return "pr_cache"; + return "core"; +} + +/** + * Aggregate raw dbstat rows into one breakdown entry per non-empty category. + * `overrides.syncBookkeeping` lets the caller replace the static + * "compactable" label with the journal-derived state (see + * `deriveSyncBookkeepingAction`) so paired projects read "compaction_pending". + */ +export function mapDbBreakdown( + rows: Array<{ name: string; bytes: number }>, + overrides?: { syncBookkeeping?: DbBreakdownEntry["action"] }, +): DbBreakdownEntry[] { + const totals = new Map(); + for (const row of rows) { + if (!row || typeof row.name !== "string") continue; + const bytes = typeof row.bytes === "number" && Number.isFinite(row.bytes) ? row.bytes : 0; + const category = classifyDbTable(row.name); + totals.set(category, (totals.get(category) ?? 0) + Math.max(0, bytes)); + } + const entries: DbBreakdownEntry[] = []; + for (const [category, bytes] of totals) { + if (bytes <= 0) continue; + const meta = DB_BREAKDOWN_META[category]; + const action = category === "sync_bookkeeping" && overrides?.syncBookkeeping !== undefined + ? overrides.syncBookkeeping + : meta.action; + entries.push({ table: meta.table, label: meta.label, bytes, category, action }); + } + return entries.sort((left, right) => right.bytes - left.bytes || left.table.localeCompare(right.table)); +} + +/** + * Sync-bookkeeping compaction state, derived without any new seam: if the most + * recent maintenance run skipped its cr-sqlite compaction because the project + * has sync peers, tombstones cannot be reclaimed yet → "compaction_pending" + * (WS-C renders this as "Waiting to compact"). Otherwise — no journal yet, or + * the last compaction actually ran / was not peer-blocked — it is "compactable". + */ +export function deriveSyncBookkeepingAction( + journal: readonly MaintenanceRunReport[], +): DbBreakdownEntry["action"] { + const lastCompact = journal[0]?.actions.find((action) => action.ledgerId === "db.operations_crsql"); + return lastCompact?.skippedReason === "has_peers" ? "compaction_pending" : "compactable"; +} + export function isObsoleteRecoveryBackup( backupPath: string, options: { projectRoot: string; db: AdeDb; now?: number }, @@ -258,7 +357,15 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti // Without the runtime's in-memory ownership signal, compression is unsafe. isPathActive: options.isPathActive ?? (() => true), }); + // `.ade/tmp` is a project-relative release-staging root (distinct from the + // `.ade/cache/tmp` layout dir) written by the /release skill and never cleaned + // by app code. `stagingTmpRoot` holds the `ade-*` system-temp staging dirs. + const adeTmpDir = path.join(layout.adeDir, "tmp"); + const stagingTmpRoot = options.stagingTmpDir ? path.resolve(options.stagingTmpDir) : os.tmpdir(); + const iosDerivedDataDir = path.join(layout.cacheDir, "ios-simulator", "DerivedData"); + const journalPath = path.join(layout.cacheDir, MAINTENANCE_JOURNAL_FILENAME); let sweepFlight: Promise | null = null; + let maintenanceFlight: Promise | null = null; let firstSweepTimer: ReturnType | null = null; let dailySweepTimer: ReturnType | null = null; @@ -271,27 +378,70 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti return sweepFlight; }; - if (options.isPathActive && options.diskPressure) { - firstSweepTimer = setTimeout(() => { - firstSweepTimer = null; - void runCompressionSweep().catch((error) => { - options.logger.warn("storage.history_sweep_failed", { - projectRoot, - error: error instanceof Error ? error.message : String(error), - }); + const isMaintenanceReport = (value: unknown): value is MaintenanceRunReport => { + if (!value || typeof value !== "object") return false; + const report = value as Record; + return typeof report.startedAt === "string" + && typeof report.finishedAt === "string" + && typeof report.trigger === "string" + && Array.isArray(report.actions) + && typeof report.reclaimedBytes === "number"; + }; + + const readMaintenanceJournal = (): MaintenanceRunReport[] => { + let raw: string; + try { + raw = fs.readFileSync(journalPath, "utf8"); + } catch (error) { + if (isEnoentError(error)) return []; + return []; + } + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed.filter(isMaintenanceReport) : []; + } catch { + return []; + } + }; + + const appendMaintenanceJournal = (report: MaintenanceRunReport): MaintenanceRunReport[] => { + const next = [report, ...readMaintenanceJournal()].slice(0, MAINTENANCE_JOURNAL_MAX_RUNS); + try { + fs.mkdirSync(path.dirname(journalPath), { recursive: true }); + fs.writeFileSync(journalPath, JSON.stringify(next, null, 2)); + } catch (error) { + options.logger.warn("storage.maintenance_journal_write_failed", { + projectRoot, + error: error instanceof Error ? error.message : String(error), }); - dailySweepTimer = setInterval(() => { - void runCompressionSweep().catch((error) => { - options.logger.warn("storage.history_sweep_failed", { - projectRoot, - error: error instanceof Error ? error.message : String(error), - }); - }); - }, HISTORY_SWEEP_INTERVAL_MS); - dailySweepTimer.unref?.(); - }, HISTORY_SWEEP_START_DELAY_MS); - firstSweepTimer.unref?.(); - } + } + return next; + }; + + const statSizeOrNull = (targetPath: string): number | null => { + try { + return fs.statSync(targetPath).size; + } catch { + return null; + } + }; + + const computeDbBreakdown = (syncBookkeepingAction: DbBreakdownEntry["action"]): DbBreakdownEntry[] => { + // dbstat is a compile-time-optional virtual table; degrade to no breakdown + // (empty list) when the SQLite build lacks it rather than failing the scan. + try { + const rows = options.db.all<{ name: string; bytes: number }>( + "select name, sum(pgsize) as bytes from dbstat group by name", + ); + return mapDbBreakdown(rows, { syncBookkeeping: syncBookkeepingAction }); + } catch (error) { + options.logger.debug("storage.db_breakdown_unavailable", { + projectRoot, + error: error instanceof Error ? error.message : String(error), + }); + return []; + } + }; const listLaneRows = (): LaneRow[] => options.db.all( "select id, name, worktree_path, archived_at from lanes", @@ -420,16 +570,16 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti add("lanes_worktrees", entry?.item); } - const tempNames = await readdirOrEmpty(os.tmpdir()); + const tempNames = await readdirOrEmpty(stagingTmpRoot); for (const name of tempNames.filter((value) => /^ade-/.test(value))) { - const tempPath = path.join(os.tmpdir(), name); + const tempPath = path.join(stagingTmpRoot, name); if (path.resolve(tempPath) === projectRoot || path.resolve(tempPath) === adeHome) continue; const stat = await lstatOrNull(tempPath); if (!stat) continue; const stale = Date.now() - stat.mtimeMs > STALE_AGE_MS; const entry = await makeItem({ category: "build_release", - base: os.tmpdir(), + base: stagingTmpRoot, path: tempPath, label: "Release and build staging", safety: stale ? "safe_to_remove" : "review_first", @@ -438,7 +588,25 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti }); add("build_release", entry?.item); } - const derivedData = path.join(layout.cacheDir, "ios-simulator", "DerivedData"); + // `.ade/tmp` direct children: /release-skill staging that nothing else reaps. + const adeTmpNames = await readdirOrEmpty(adeTmpDir); + for (const name of adeTmpNames) { + const adeTmpPath = path.join(adeTmpDir, name); + const stat = await lstatOrNull(adeTmpPath); + if (!stat || stat.isSymbolicLink()) continue; + const stale = Date.now() - stat.mtimeMs > STALE_AGE_MS; + const entry = await makeItem({ + category: "build_release", + base: adeTmpDir, + path: adeTmpPath, + label: "Release staging", + safety: stale ? "safe_to_remove" : "review_first", + state, + detail: stale ? "Old release staging is no longer in use" : "A current release may still be using these files", + }); + add("build_release", entry?.item); + } + const derivedData = iosDerivedDataDir; add("build_release", (await makeItem({ category: "build_release", base: layout.adeDir, @@ -534,6 +702,19 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti buildCategory("recovery_backups", categoryItems.get("recovery_backups") ?? [], "review_first", state), buildCategory("database", categoryItems.get("database") ?? [], "protected", state), ]; + let safeReclaimableBytes = compressibleBytes; + for (const items of categoryItems.values()) { + for (const item of items) { + if (item.safety === "safe_to_remove") safeReclaimableBytes += item.bytes; + } + } + const journal = readMaintenanceJournal(); + const extras: StorageSnapshotExtras = { + dbBreakdown: computeDbBreakdown(deriveSyncBookkeepingAction(journal)), + maintenance: { lastRun: journal[0] ?? null, journal }, + safeReclaimableBytes, + policyChips: deriveCategoryPolicyChips(), + }; const snapshot: StorageSnapshot = { generatedAt: new Date().toISOString(), projectRoot, @@ -542,6 +723,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti categories, scanDurationMs: Date.now() - startedAt, truncated: state.truncated, + extras, }; if (state.truncated) { options.logger.warn("storage.scan_truncated", { projectRoot, entries: state.entries, entryLimit: state.entryLimit, budgetMs: state.budgetMs }); @@ -576,9 +758,17 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti } label = lane?.name ?? path.basename(targetPath); } else if (target.kind === "stale_tmp_staging") { - if (!isDirectChild(os.tmpdir(), targetPath) || !/^ade-/.test(path.basename(targetPath))) { + // Two staging roots share this kind: `ade-*` dirs in the system temp root, + // and any direct child of the project-relative `.ade/tmp` release-staging + // dir. Direct-child containment is the safety boundary for both. + const inSystemStaging = isDirectChild(stagingTmpRoot, targetPath) && /^ade-/.test(path.basename(targetPath)); + const inProjectStaging = isDirectChild(adeTmpDir, targetPath); + if (!inSystemStaging && !inProjectStaging) { return { valid: null, reason: "This path is not ADE staging data." }; } + if (inProjectStaging && await hasSymlinkAncestor(projectRoot, targetPath)) { + return { valid: null, reason: "Links cannot be used in a cleanup path." }; + } if (targetPath === projectRoot || targetPath === adeHome) { return { valid: null, reason: "Project data is protected." }; } @@ -720,6 +910,230 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti return { filesCompressed: result.filesCompressed, savedBytes: result.savedBytes }; }; + // Reap a set of candidate targets through the same validate/preview/cleanup + // pipeline the manual flow uses. Only targets the validator lets into the + // preview are removed, so protected/review_first/fresh items are never touched. + const reapTargets = async ( + targets: StorageCleanupTarget[], + ): Promise<{ itemsAffected: number; bytesReclaimed: number }> => { + if (targets.length === 0) return { itemsAffected: 0, bytesReclaimed: 0 }; + const preview = await cleanupPreview(targets); + if (preview.items.length === 0) return { itemsAffected: 0, bytesReclaimed: 0 }; + const previewed = new Set(preview.items.map((item) => path.resolve(item.path))); + const removable = targets.filter((target) => previewed.has(path.resolve(target.path))); + const result = await cleanup(removable, { preview }); + return { itemsAffected: result.removed.length, bytesReclaimed: result.freedBytes }; + }; + + const collectSystemStaging = async (): Promise => { + const names = await readdirOrEmpty(stagingTmpRoot); + const targets: StorageCleanupTarget[] = []; + for (const name of names.filter((value) => /^ade-/.test(value))) { + const tempPath = path.join(stagingTmpRoot, name); + if (path.resolve(tempPath) === projectRoot || path.resolve(tempPath) === adeHome) continue; + const stat = await lstatOrNull(tempPath); + if (!stat || stat.isSymbolicLink()) continue; + if (Date.now() - stat.mtimeMs <= STALE_AGE_MS) continue; + targets.push({ kind: "stale_tmp_staging", path: tempPath }); + } + return targets; + }; + + const collectProjectStaging = async (): Promise => { + const names = await readdirOrEmpty(adeTmpDir); + const targets: StorageCleanupTarget[] = []; + for (const name of names) { + const tempPath = path.join(adeTmpDir, name); + const stat = await lstatOrNull(tempPath); + if (!stat || stat.isSymbolicLink()) continue; + if (Date.now() - stat.mtimeMs <= STALE_AGE_MS) continue; + targets.push({ kind: "stale_tmp_staging", path: tempPath }); + } + return targets; + }; + + const collectObsoleteBackups = async (): Promise => { + const names = await readdirOrEmpty(layout.adeDir); + const targets: StorageCleanupTarget[] = []; + for (const name of names.filter((value) => RECOVERY_BACKUP_PATTERN.test(value))) { + const backupPath = path.join(layout.adeDir, name); + if (isObsoleteRecoveryBackup(backupPath, { projectRoot, db: options.db })) { + targets.push({ kind: "recovery_backup", path: backupPath }); + } + } + return targets; + }; + + const collectIosDerivedData = async (): Promise => { + const stat = await lstatOrNull(iosDerivedDataDir); + if (!stat || stat.isSymbolicLink() || !stat.isDirectory()) return []; + return [{ kind: "rebuildable_cache", path: iosDerivedDataDir }]; + }; + + // Every step is independently try/caught so one failure never aborts the run. + const runStep = async ( + actions: MaintenanceAction[], + ledgerId: string, + kind: MaintenanceActionKind, + fn: () => Promise<{ itemsAffected: number; bytesReclaimed: number; skippedReason?: string | null }>, + ): Promise => { + const start = Date.now(); + try { + const result = await fn(); + actions.push({ + ledgerId, + kind, + itemsAffected: result.itemsAffected, + bytesReclaimed: result.bytesReclaimed, + durationMs: Date.now() - start, + skippedReason: result.skippedReason ?? null, + error: null, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + actions.push({ + ledgerId, + kind, + itemsAffected: 0, + bytesReclaimed: 0, + durationMs: Date.now() - start, + skippedReason: null, + error: message, + }); + options.logger.warn("storage.maintenance_step_failed", { projectRoot, ledgerId, kind, error: message }); + } + }; + + const runMaintenanceSweep = (trigger: MaintenanceTrigger): Promise => { + if (maintenanceFlight) return maintenanceFlight; + const start = async (): Promise => { + const startedAt = new Date().toISOString(); + const actions: MaintenanceAction[] = []; + + // a. Compress inactive chat/terminal history (existing safe mechanics). + await runStep(actions, "fs.transcripts", "compress", async () => { + const summary = await runCompressionSweep(); + return { itemsAffected: summary.filesCompressed, bytesReclaimed: summary.savedBytes }; + }); + + // b/c. Auto-reap safe_to_remove staging, obsolete backups, and iOS build data. + await runStep(actions, "fs.tmp_staging", "delete", async () => reapTargets(await collectSystemStaging())); + await runStep(actions, "fs.tmp", "delete", async () => reapTargets(await collectProjectStaging())); + await runStep(actions, "fs.recovery_backups", "delete", async () => reapTargets(await collectObsoleteBackups())); + await runStep(actions, "fs.ios_derived_data", "delete", async () => reapTargets(await collectIosDerivedData())); + + // d. DB retention hooks. `maintenance` is attached by WS-A's kvDb; until it + // lands the hooks are undefined and each records an "unsupported" skip. + const maintenance = (options.db as unknown as KvDbWithMaintenance).maintenance; + const runDbStep = ( + ledgerId: string, + kind: MaintenanceActionKind, + fn: (() => { itemsAffected: number; bytesReclaimed: number; skippedReason?: string | null }) | undefined, + ): Promise => + runStep(actions, ledgerId, kind, async () => { + if (!fn) return { itemsAffected: 0, bytesReclaimed: 0, skippedReason: "unsupported" }; + const result = fn(); + return { + itemsAffected: result.itemsAffected, + bytesReclaimed: result.bytesReclaimed, + skippedReason: result.skippedReason ?? null, + }; + }); + await runDbStep("db.automation_ingress_events", "prune", maintenance?.pruneIngressEvents.bind(maintenance)); + await runDbStep("db.review_run_artifacts", "prune", maintenance?.pruneReviewArtifacts.bind(maintenance)); + await runDbStep("db.pull_request_snapshots", "prune", maintenance?.prunePrSnapshots.bind(maintenance)); + await runDbStep("db.operations_crsql", "compact", maintenance?.compactCrsqlTombstones.bind(maintenance)); + await runDbStep( + "db.core", + "vacuum", + maintenance ? () => maintenance.vacuumIfFragmented(VACUUM_FREELIST_THRESHOLD) : undefined, + ); + + const finishedAt = new Date().toISOString(); + const reclaimedBytes = actions.reduce((sum, action) => sum + Math.max(0, action.bytesReclaimed), 0); + const report: MaintenanceRunReport = { + startedAt, + finishedAt, + trigger, + actions, + reclaimedBytes, + dbSizeBytes: statSizeOrNull(layout.dbPath), + }; + appendMaintenanceJournal(report); + cachedSnapshot = null; + + const failedCount = actions.filter((action) => action.error).length; + const outcome = failedCount === 0 ? "completed" : failedCount >= actions.length ? "failed" : "partial"; + const filesCompressed = actions.find((action) => action.ledgerId === "fs.transcripts")?.itemsAffected ?? 0; + options.logger.info("storage.maintenance_completed", { + projectRoot, + trigger, + outcome, + reclaimedBytes, + filesCompressed, + failedSteps: failedCount, + durationMs: Date.parse(finishedAt) - Date.parse(startedAt), + }); + try { + options.captureAnalytics?.({ + event: "ade_feature_used", + surface: "desktop", + properties: { + feature: "storage_doctor", + action: "maintenance_run", + outcome, + bytes_freed: reclaimedBytes, + files_compressed: filesCompressed, + }, + projectId: options.projectId ?? null, + // Local-only dedupe fingerprint (hashed before send): collapses repeat + // runs within 20 h into one PostHog event. + dedupeKey: `storage_doctor_run:${options.projectId ?? projectRoot}`, + minimumIntervalMs: MAINTENANCE_ANALYTICS_MIN_INTERVAL_MS, + }); + } catch (error) { + options.logger.debug("storage.maintenance_analytics_failed", { + projectRoot, + error: error instanceof Error ? error.message : String(error), + }); + } + return report; + }; + maintenanceFlight = start().finally(() => { + maintenanceFlight = null; + }); + return maintenanceFlight; + }; + + const runMaintenanceNow = (): Promise => runMaintenanceSweep("manual"); + + // The daemon-backed fallback instance (isPathActive without diskPressure) must + // never schedule automatic maintenance; only the real daemon instance, which + // supplies both signals, arms the doctor's post-boot + daily timers. + if (options.isPathActive && options.diskPressure) { + firstSweepTimer = setTimeout(() => { + firstSweepTimer = null; + void runMaintenanceSweep("post_boot").catch((error) => { + options.logger.warn("storage.maintenance_failed", { + projectRoot, + trigger: "post_boot", + error: error instanceof Error ? error.message : String(error), + }); + }); + dailySweepTimer = setInterval(() => { + void runMaintenanceSweep("daily").catch((error) => { + options.logger.warn("storage.maintenance_failed", { + projectRoot, + trigger: "daily", + error: error instanceof Error ? error.message : String(error), + }); + }); + }, HISTORY_SWEEP_INTERVAL_MS); + dailySweepTimer.unref?.(); + }, HISTORY_SWEEP_START_DELAY_MS); + firstSweepTimer.unref?.(); + } + const dispose = (): void => { if (firstSweepTimer) clearTimeout(firstSweepTimer); if (dailySweepTimer) clearInterval(dailySweepTimer); @@ -727,5 +1141,5 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti dailySweepTimer = null; }; - return { getSnapshot, cleanupPreview, cleanup, compressNow, dispose }; + return { getSnapshot, cleanupPreview, cleanup, compressNow, runMaintenanceNow, dispose }; } diff --git a/apps/desktop/src/main/services/storage/storageLedger.test.ts b/apps/desktop/src/main/services/storage/storageLedger.test.ts new file mode 100644 index 000000000..8b1bf47b3 --- /dev/null +++ b/apps/desktop/src/main/services/storage/storageLedger.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { ADE_LAYOUT_DEFINITIONS } from "../../../shared/adeLayout"; +import { + deriveCategoryPolicyChips, + getLedgerEntry, + LEDGER_LAYOUT_COVERAGE, + STORAGE_LEDGER, +} from "./storageLedger"; + +describe("storageLedger", () => { + it("declares a well-formed, unique policy for every entry", () => { + const ids = new Set(); + for (const entry of STORAGE_LEDGER) { + expect(entry.id).toMatch(/^[a-z]+\.[a-z0-9_]+$/); + expect(ids.has(entry.id)).toBe(false); + ids.add(entry.id); + expect(entry.description.length).toBeGreaterThan(0); + expect(["table", "directory", "file_family"]).toContain(entry.kind); + expect(["user_data", "derived", "operational"]).toContain(entry.policyClass); + expect(["write_time", "doctor", "both", "manual"]).toContain(entry.enforcement); + // compressAfterDays only applies to user_data (spec §2 contract). + if (entry.policy.compressAfterDays != null) { + expect(entry.policyClass).toBe("user_data"); + } + } + }); + + it("covers every §1 evidence table and the doctor's own artifacts", () => { + for (const id of [ + "db.automation_ingress_events", + "db.operations_crsql", + "db.review_run_artifacts", + "db.pull_request_snapshots", + "fs.transcripts", + "fs.tmp", + "fs.recovery_backups", + "fs.cache", + "fs.storage_doctor_journal", + "fs.artifacts", + "fs.attachments", + ]) { + expect(getLedgerEntry(id), `missing ledger entry ${id}`).toBeDefined(); + } + }); + + it("declares a storage policy (or explicit unmanaged waiver) for every layout directory", () => { + // CI guard: adding a new tracked directory to ADE_LAYOUT_DEFINITIONS without + // declaring its storage policy here must fail. Either map it to a ledger id + // or add it to LEDGER_LAYOUT_COVERAGE as intentionally unmanaged (null). + const ledgerIds = new Set(STORAGE_LEDGER.map((entry) => entry.id)); + for (const definition of ADE_LAYOUT_DEFINITIONS) { + if (definition.pathType !== "directory") continue; + const declared = Object.prototype.hasOwnProperty.call( + LEDGER_LAYOUT_COVERAGE, + definition.relativePath, + ); + expect(declared, `layout dir ${definition.relativePath} has no ledger coverage entry`).toBe(true); + const ledgerId = LEDGER_LAYOUT_COVERAGE[definition.relativePath]; + if (ledgerId !== null && ledgerId !== undefined) { + expect(ledgerIds.has(ledgerId), `coverage points at missing ledger id ${ledgerId}`).toBe(true); + } + } + }); + + it("derives category policy chips from the ledger policy values", () => { + const chips = deriveCategoryPolicyChips(); + expect(chips.chats_history).toBe("Compressed after 14 days"); + expect(chips.build_release).toBe("Auto-cleans after 7 days"); + expect(chips.recovery_backups).toBe("Keeps the latest backup"); + expect(chips.caches).toBeTruthy(); + expect(chips.lanes_worktrees).toBeTruthy(); + }); +}); diff --git a/apps/desktop/src/main/services/storage/storageLedger.ts b/apps/desktop/src/main/services/storage/storageLedger.ts new file mode 100644 index 000000000..88ba6dc1e --- /dev/null +++ b/apps/desktop/src/main/services/storage/storageLedger.ts @@ -0,0 +1,208 @@ +// Storage ledger: the declared policy for every persistent table and directory +// ADE writes. Each entry states what the data is, its privacy class, and how it +// is bounded (write-time cap, storage-doctor sweep, or manual only). The ledger +// is the single source of truth the Settings UI reads for its policy chips and +// that CI cross-checks against `ADE_LAYOUT_DEFINITIONS` so a new tracked layout +// directory cannot ship without a declared storage policy (see +// `storageLedger.test.ts`). + +import type { + StorageCategoryId, + StorageLedgerEntry, +} from "../../../shared/types/storage"; + +export const STORAGE_LEDGER: readonly StorageLedgerEntry[] = [ + // --- Project database tables (§1 evidence) ------------------------------- + { + id: "db.automation_ingress_events", + kind: "table", + description: "Webhook history — inbound automation events. 99.99% are written and never read.", + policyClass: "operational", + policy: { maxAgeDays: 7, maxRows: 2_000 }, + enforcement: "both", + }, + { + id: "db.operations_crsql", + kind: "table", + description: "Sync bookkeeping — cr-sqlite change-tracking clock and primary-key shadow rows.", + policyClass: "operational", + policy: {}, + enforcement: "doctor", + }, + { + id: "db.review_run_artifacts", + kind: "table", + description: "Review artifacts — cached code-review results, re-derivable from a fresh run.", + policyClass: "derived", + policy: { maxAgeDays: 30 }, + enforcement: "both", + }, + { + id: "db.pull_request_snapshots", + kind: "table", + description: "PR cache — pull-request metadata re-fetchable from GitHub.", + policyClass: "derived", + policy: { maxAgeDays: 60 }, + enforcement: "both", + }, + { + id: "db.core", + kind: "table", + description: "Core project data — chats, lanes, and other durable records. Never auto-deleted.", + policyClass: "user_data", + policy: {}, + enforcement: "manual", + }, + // --- Filesystem directories / file families ------------------------------ + { + id: "fs.transcripts", + kind: "directory", + description: "Chat and terminal history. Inactive files are gzip-compressed with transparent read-back.", + policyClass: "user_data", + policy: { compressAfterDays: 14 }, + enforcement: "doctor", + }, + { + id: "fs.tmp", + kind: "directory", + description: "Project release staging under .ade/tmp (TestFlight and release verification scratch).", + policyClass: "derived", + policy: { maxAgeDays: 7 }, + enforcement: "doctor", + }, + { + id: "fs.tmp_staging", + kind: "file_family", + description: "System temp staging (ade-* directories) left by build and release operations.", + policyClass: "derived", + policy: { maxAgeDays: 7 }, + enforcement: "doctor", + }, + { + id: "fs.recovery_backups", + kind: "file_family", + description: "Database recovery backups. The doctor keeps the newest good copy and reaps obsolete ones.", + policyClass: "operational", + policy: { keepLatest: 1, maxAgeDays: 7 }, + enforcement: "doctor", + }, + { + id: "fs.cache", + kind: "directory", + description: "Rebuildable caches under .ade/cache. Recreated on demand.", + policyClass: "derived", + policy: {}, + enforcement: "doctor", + }, + { + id: "fs.ios_derived_data", + kind: "directory", + description: "iOS simulator DerivedData build cache. Recreated the next time you build.", + policyClass: "derived", + policy: {}, + enforcement: "doctor", + }, + { + id: "fs.storage_doctor_journal", + kind: "file_family", + description: "Storage maintenance journal recording the last 30 doctor runs.", + policyClass: "operational", + policy: { keepLatest: 30 }, + enforcement: "doctor", + }, + { + id: "fs.artifacts", + kind: "directory", + description: "Proof, recordings, and captured artifacts. Kept until you remove them.", + policyClass: "user_data", + policy: {}, + enforcement: "manual", + }, + { + id: "fs.attachments", + kind: "directory", + description: "Chat attachments. Kept until you remove them.", + policyClass: "user_data", + policy: {}, + enforcement: "manual", + }, + { + id: "fs.worktrees", + kind: "directory", + description: "Lane worktrees. Active lanes are protected; archived and orphaned ones are removed only on request.", + policyClass: "user_data", + policy: {}, + enforcement: "manual", + }, +] as const; + +const _byId = new Map(STORAGE_LEDGER.map((entry) => [entry.id, entry])); + +export function getLedgerEntry(id: string): StorageLedgerEntry | undefined { + return _byId.get(id); +} + +/** + * Maps every `ADE_LAYOUT_DEFINITIONS` directory entry to a ledger id, or `null` + * when the directory is intentionally not storage-managed (user-authored config + * or credentials the doctor must never touch). The coverage test asserts that + * every layout directory appears here, so adding a new tracked directory + * without declaring its storage policy fails CI. + */ +export const LEDGER_LAYOUT_COVERAGE: Readonly> = { + // Storage-managed directories. + transcripts: "fs.transcripts", + cache: "fs.cache", + worktrees: "fs.worktrees", + artifacts: "fs.artifacts", + // Intentionally unmanaged: git-tracked, user-authored shared assets. These are + // small config/source directories the storage doctor deliberately never sweeps. + cto: null, + templates: null, + skills: null, + workflows: null, + "workflows/linear": null, + "project-icons": null, + // Intentionally unmanaged: agent-runtime scratch owned by the runtimes that + // write them. Small and short-lived; not doctor-managed in this release. + agents: null, + context: null, + history: null, + reflections: null, + // Intentionally unmanaged: credentials. Never auto-managed by storage tooling. + secrets: null, +}; + +/** + * Human-readable policy chips per storage category, derived from the ledger so a + * policy change (e.g. compress-after days) updates the Settings copy in lockstep. + * Copy rules: sentence-case, verbs first, no internal jargon. + */ +export function deriveCategoryPolicyChips(): Partial> { + const chips: Partial> = {}; + + const transcripts = _byId.get("fs.transcripts"); + if (transcripts?.policy.compressAfterDays != null) { + chips.chats_history = `Compressed after ${transcripts.policy.compressAfterDays} days`; + } + + const tmp = _byId.get("fs.tmp"); + if (tmp?.policy.maxAgeDays != null) { + chips.build_release = `Auto-cleans after ${tmp.policy.maxAgeDays} days`; + } + + chips.caches = "Rebuilt when needed"; + + const backups = _byId.get("fs.recovery_backups"); + if (backups?.policy.keepLatest != null) { + chips.recovery_backups = backups.policy.keepLatest === 1 + ? "Keeps the latest backup" + : `Keeps the newest ${backups.policy.keepLatest}`; + } + + chips.lanes_worktrees = "Kept until you remove them"; + chips.proof_attachments = "Kept until you remove them"; + chips.database = "Cleaned during maintenance"; + + return chips; +} diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index 41720f151..9c1516c82 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -669,6 +669,8 @@ import type { } from "../shared/types"; import type { DiskPressureSnapshot } from "../main/services/storage/diskPressure"; import type { + MaintenanceRunReport, + RuntimeHealthSnapshot, StorageCleanupPreview, StorageCleanupResult, StorageCleanupTarget, @@ -698,6 +700,7 @@ declare global { ping: () => Promise<"pong">; getInfo: () => Promise; getResourceUsage: () => Promise; + getRuntimeHealth: () => Promise; getLatestRelease: () => Promise; getProject: () => Promise; getWindowSession: () => Promise<{ @@ -759,6 +762,7 @@ declare global { getPressure: () => Promise; getSnapshot: (args?: { forceRefresh?: boolean }) => Promise; compressNow: () => Promise; + runMaintenanceNow: () => Promise; cleanupPreview: (targets: StorageCleanupTarget[]) => Promise; cleanup: ( targets: StorageCleanupTarget[], diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 431abbb52..c12f63571 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -13,6 +13,8 @@ import type { } from "../shared/types/productAnalytics"; import type { DiskPressureSnapshot } from "../main/services/storage/diskPressure"; import type { + MaintenanceRunReport, + RuntimeHealthSnapshot, StorageCleanupPreview, StorageCleanupResult, StorageCleanupTarget, @@ -3236,6 +3238,8 @@ contextBridge.exposeInMainWorld("ade", { getInfo: async (): Promise => ipcRenderer.invoke(IPC.appGetInfo), getResourceUsage: async (): Promise => ipcRenderer.invoke(IPC.appGetResourceUsage), + getRuntimeHealth: async (): Promise => + ipcRenderer.invoke(IPC.appGetRuntimeHealth), getLatestRelease: async (): Promise => ipcRenderer.invoke(IPC.appGetLatestRelease), getProject: async (): Promise => @@ -3376,6 +3380,10 @@ contextBridge.exposeInMainWorld("ade", { callProjectRuntimeActionOr("storage", "compressNow", { args: {} }, () => ipcRenderer.invoke(IPC.storageCompressNow), ), + runMaintenanceNow: async (): Promise => + callProjectRuntimeActionOr("storage", "runMaintenanceNow", {}, () => + ipcRenderer.invoke(IPC.storageRunMaintenanceNow), + ), cleanupPreview: async (targets: StorageCleanupTarget[]): Promise => callProjectRuntimeActionOr("storage", "cleanupPreview", { args: { targets } }, () => ipcRenderer.invoke(IPC.storageCleanupPreview, targets), diff --git a/apps/desktop/src/renderer/browserMock.ts b/apps/desktop/src/renderer/browserMock.ts index b3b3d8b8d..1230f9f5e 100644 --- a/apps/desktop/src/renderer/browserMock.ts +++ b/apps/desktop/src/renderer/browserMock.ts @@ -3335,8 +3335,16 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { ptyProcessCount: 0, ptyCpuPercent: 0, ptyMemoryMB: 0, - freeMemoryMB: null, - totalMemoryMB: null, + freeMemoryMB: 8_000, + totalMemoryMB: 16_000, + roleUsage: [ + { role: "ade-runtime", processCount: 1, cpuPercent: 2, memoryMB: 280 }, + ], + }), + getRuntimeHealth: resolved({ + slowActions24h: 0, + slowActionP95Ms: null, + sampledAt: now, }), getLatestRelease: resolved({ version: "1.0.0", @@ -3412,10 +3420,51 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { categories: [], scanDurationMs: 0, truncated: false, + extras: { + dbBreakdown: [ + { table: "automation_ingress_events", label: "Webhook history", bytes: 12 * 1024 ** 2, category: "webhooks", action: "prunable" }, + { table: "operations", label: "Sync bookkeeping", bytes: 6 * 1024 ** 2, category: "sync_bookkeeping", action: "compactable" }, + { table: "core", label: "Core data", bytes: 8 * 1024 ** 2, category: "core", action: null }, + ], + maintenance: { + lastRun: { + startedAt: now, + finishedAt: now, + trigger: "daily", + actions: [], + reclaimedBytes: 0, + dbSizeBytes: 26 * 1024 ** 2, + }, + journal: [ + { + startedAt: now, + finishedAt: now, + trigger: "daily", + actions: [], + reclaimedBytes: 0, + dbSizeBytes: 26 * 1024 ** 2, + }, + ], + }, + safeReclaimableBytes: 18 * 1024 ** 2, + policyChips: { + chats_history: "Compressed after 14 days", + build_release: "Auto-cleans · 7 days", + caches: "Rebuilt on demand", + }, + }, }), cleanupPreview: resolvedArg({ items: [], totalBytes: 0, blocked: [] }), compressNow: resolvedArg({ filesCompressed: 0, savedBytes: 0 }), cleanup: resolvedArg({ removed: [], failed: [], freedBytes: 0 }), + runMaintenanceNow: resolved({ + startedAt: now, + finishedAt: now, + trigger: "manual" as const, + actions: [], + reclaimedBytes: 18 * 1024 ** 2, + dbSizeBytes: 20 * 1024 ** 2, + }), }, project: { openRepo: resolved(MOCK_PROJECT), diff --git a/apps/desktop/src/renderer/components/app/SettingsPage.tsx b/apps/desktop/src/renderer/components/app/SettingsPage.tsx index 78f93cbcd..7c5e3b1c3 100644 --- a/apps/desktop/src/renderer/components/app/SettingsPage.tsx +++ b/apps/desktop/src/renderer/components/app/SettingsPage.tsx @@ -54,6 +54,7 @@ const TAB_ALIASES: Record = { const HASH_TARGET_SECTIONS: Partial> = { "ai-providers": "ai", + diagnostics: "storage", secrets: "secrets", "chat-launch-clipboard": "general", "agent-completion-sound": "general", diff --git a/apps/desktop/src/renderer/components/app/TopBar.tsx b/apps/desktop/src/renderer/components/app/TopBar.tsx index 6eb1ff151..25bbba3d3 100644 --- a/apps/desktop/src/renderer/components/app/TopBar.tsx +++ b/apps/desktop/src/renderer/components/app/TopBar.tsx @@ -290,9 +290,9 @@ function ResourcePressureIndicator({ usage }: { usage: AppResourceUsageSnapshot }} wrapperStyle={{ WebkitAppRegion: "no-drag" } as React.CSSProperties} > - { + window.location.hash = "#/settings?tab=storage#diagnostics"; + }} > - + ); } diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx index f49276343..102baed57 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx @@ -5,11 +5,15 @@ import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-li import { expectNoJargon } from "../../../test/jargonGuard"; import { StorageSection } from "./StorageSection"; import type { + MaintenanceRunReport, + RuntimeHealthSnapshot, StorageCleanupPreview, StorageCleanupResult, StorageCleanupTarget, StorageSnapshot, + StorageSnapshotExtras, } from "../../../shared/types/storage"; +import type { AppResourceUsageSnapshot } from "../../../shared/types"; const originalAde = (globalThis.window as any)?.ade; @@ -98,7 +102,89 @@ function makeSnapshot(): StorageSnapshot { }; } -function installAdeMock(options: { withCompress?: boolean } = {}) { +const MB = 1024 ** 2; + +function makeExtras(): StorageSnapshotExtras { + const nowIso = new Date().toISOString(); + const yesterdayIso = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + return { + dbBreakdown: [ + { table: "automation_ingress_events", label: "Webhook history", bytes: 40 * MB, category: "webhooks", action: "prunable" }, + { table: "operations", label: "Sync bookkeeping", bytes: 20 * MB, category: "sync_bookkeeping", action: "compactable" }, + { table: "pull_request_snapshots", label: "Pull request cache", bytes: 4 * MB, category: "pr_cache", action: "compaction_pending" }, + { table: "core", label: "Core data", bytes: 8 * MB, category: "core", action: null }, + ], + maintenance: { + lastRun: { + startedAt: yesterdayIso, + finishedAt: yesterdayIso, + trigger: "daily", + actions: [ + { ledgerId: "automation.ingress_events", kind: "prune", itemsAffected: 200, bytesReclaimed: 450 * MB, durationMs: 40 }, + { ledgerId: "operations", kind: "compact", itemsAffected: 0, bytesReclaimed: 30 * MB, durationMs: 60 }, + ], + reclaimedBytes: 480 * MB, + dbSizeBytes: 60 * MB, + }, + journal: [ + { + startedAt: yesterdayIso, + finishedAt: yesterdayIso, + trigger: "daily", + actions: [ + { ledgerId: "automation.ingress_events", kind: "prune", itemsAffected: 200, bytesReclaimed: 450 * MB, durationMs: 40 }, + { ledgerId: "operations", kind: "compact", itemsAffected: 0, bytesReclaimed: 30 * MB, durationMs: 60 }, + ], + reclaimedBytes: 480 * MB, + dbSizeBytes: 60 * MB, + }, + { + startedAt: nowIso, + finishedAt: nowIso, + trigger: "manual", + actions: [], + reclaimedBytes: 0, + dbSizeBytes: 32 * MB, + }, + ], + }, + safeReclaimableBytes: 460 * MB, + policyChips: { + chats_history: "Compressed after 14 days", + build_release: "Auto-cleans · 7 days", + caches: "Rebuilt on demand", + }, + }; +} + +function makeSnapshotWithExtras(): StorageSnapshot { + return { ...makeSnapshot(), extras: makeExtras() }; +} + +function makeUsage(): AppResourceUsageSnapshot { + return { + sampledAt: new Date().toISOString(), + processCount: 4, + cpuPercent: 2, + mainCpuPercent: 1, + rendererCpuPercent: 1, + memoryMB: 500, + mainMemoryMB: 200, + rendererMemoryMB: 300, + activePtyCount: 0, + ptyProcessCount: 0, + ptyCpuPercent: null, + ptyMemoryMB: null, + freeMemoryMB: 8_000, + totalMemoryMB: 16_000, + roleUsage: [ + { role: "ade-runtime", processCount: 1, cpuPercent: 2, memoryMB: 280 }, + { role: "electron-main", processCount: 1, cpuPercent: 3, memoryMB: 500 }, + ], + } as AppResourceUsageSnapshot; +} + +function installAdeMock(options: { withCompress?: boolean; withExtras?: boolean; withApp?: boolean } = {}) { const cleanupPreview = vi.fn( async (targets: StorageCleanupTarget[]): Promise => ({ items: targets.map((target) => ({ path: target.path, bytes: 1.5 * 1024 ** 3, label: "Old feature" })), @@ -115,15 +201,33 @@ function installAdeMock(options: { withCompress?: boolean } = {}) { async (_targets: StorageCleanupTarget[], _opts: { preview: StorageCleanupPreview }) => cleanupResult, ); const compressNow = vi.fn(async () => ({ filesCompressed: 12, savedBytes: 300 * 1024 ** 2 })); + const runMaintenanceNow = vi.fn( + async (): Promise => ({ + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + trigger: "manual", + actions: [ + { ledgerId: "automation.ingress_events", kind: "prune", itemsAffected: 200, bytesReclaimed: 450 * MB, durationMs: 40 }, + ], + reclaimedBytes: 481 * MB, + dbSizeBytes: 30 * MB, + }), + ); + const getRuntimeHealth = vi.fn( + async (): Promise => ({ slowActions24h: 3, slowActionP95Ms: 5_200, sampledAt: new Date().toISOString() }), + ); + const getResourceUsage = vi.fn(async () => makeUsage()); const storage: Record = { - getSnapshot: vi.fn(async () => makeSnapshot()), + getSnapshot: vi.fn(async () => (options.withExtras ? makeSnapshotWithExtras() : makeSnapshot())), getPressure: vi.fn(async () => ({ state: "normal", freeBytes: 40 * 1024 ** 3, totalBytes: 500 * 1024 ** 3, freeFraction: 0.08, perRoot: [], sampledAt: new Date().toISOString() })), cleanupPreview, cleanup: cleanupFn, }; if (options.withCompress) storage.compressNow = compressNow; + if (options.withExtras) storage.runMaintenanceNow = runMaintenanceNow; + const includeApp = options.withApp ?? options.withExtras; (globalThis.window as any).ade = { storage, lanes: { @@ -132,8 +236,9 @@ function installAdeMock(options: { withCompress?: boolean } = {}) { { id: "lane-main", name: "main-lane", worktreePath: ACTIVE_PATH, archivedAt: null, status: { dirty: false } }, ]), }, + ...(includeApp ? { app: { getResourceUsage, getRuntimeHealth } } : {}), }; - return { cleanupPreview, cleanup: cleanupFn, compressNow }; + return { cleanupPreview, cleanup: cleanupFn, compressNow, runMaintenanceNow, getRuntimeHealth, getResourceUsage }; } describe("StorageSection", () => { @@ -236,4 +341,69 @@ describe("StorageSection", () => { await screen.findByText(/ADE is using/); expectNoJargon(container.textContent ?? ""); }); + + it("hides the safe-cleanup primary when the daemon reports no reclaimable space", async () => { + installAdeMock(); + render(); + await screen.findByText(/ADE is using/); + expect(screen.queryByRole("button", { name: /clean up safely/i })).toBeNull(); + }); + + it("runs the doctor from the safe-cleanup primary and reports what it reclaimed", async () => { + const { runMaintenanceNow } = installAdeMock({ withExtras: true }); + render(); + + const primary = await screen.findByRole("button", { name: /clean up safely/i }); + fireEvent.click(primary); + + const dialog = await screen.findByRole("dialog"); + const confirm = await within(dialog).findByRole("button", { name: /clean up safely/i }); + fireEvent.click(confirm); + + await waitFor(() => expect(runMaintenanceNow).toHaveBeenCalledTimes(1)); + expect(await within(dialog).findByText(/Freed 481 MB/)).toBeTruthy(); + }); + + it("shows the database breakdown and runs maintenance from an inline action", async () => { + const { runMaintenanceNow } = installAdeMock({ withExtras: true }); + render(); + + // Human-labeled breakdown rows replace the blanket "Protected" body. + expect(await screen.findByText("Webhook history")).toBeTruthy(); + expect(screen.getByText("Sync bookkeeping")).toBeTruthy(); + expect(screen.getByText("Core data")).toBeTruthy(); + // Pending compaction is surfaced without jargon and without an action. + expect(screen.getByText(/Waiting to compact/)).toBeTruthy(); + + const compact = screen.getByRole("button", { name: /compact now/i }); + fireEvent.click(compact); + await waitFor(() => expect(runMaintenanceNow).toHaveBeenCalledTimes(1)); + }); + + it("renders diagnostics from resource usage and the maintenance journal", async () => { + installAdeMock({ withExtras: true }); + render(); + + expect(await screen.findByText(/Health & diagnostics/)).toBeTruthy(); + // Daemon resident memory from the ade-runtime role. + expect(await screen.findByText("280 MB")).toBeTruthy(); + // Overall health chip from the resource-pressure level. + expect(await screen.findByText("Healthy")).toBeTruthy(); + // Slow-actions tile from getRuntimeHealth. + expect(screen.getByText(/3 slow responses in 24h/)).toBeTruthy(); + + // Journal expands to show a humanized run summary. + fireEvent.click(screen.getByRole("button", { name: /recent cleanups/i })); + expect(await screen.findByText(/reclaimed 450 MB/)).toBeTruthy(); + }); + + it("keeps plain language across the diagnostics and database surfaces", async () => { + installAdeMock({ withExtras: true }); + const { container } = render(); + await screen.findByText("Webhook history"); + fireEvent.click(screen.getByRole("button", { name: /recent cleanups/i })); + fireEvent.click(await screen.findByRole("button", { name: /clean up safely/i })); + await screen.findByRole("dialog"); + expectNoJargon(container.textContent ?? ""); + }); }); diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.tsx index 7bd641a01..ae6f07916 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.tsx @@ -2,38 +2,79 @@ import React from "react"; import { Archive, ArrowClockwise, + ArrowDown, + ArrowUp, Broom, CaretDown, CaretRight, + ChartLineUp, + Clock, FileZip, FolderDashed, + Gauge, HardDrives, + Pulse, ShieldCheck, + WarningCircle, } from "@phosphor-icons/react"; import { isUrgentDiskPressure, type DiskPressureSnapshot, + type MaintenanceRunReport, + type RuntimeHealthSnapshot, type StorageCategoryId, type StorageCategorySnapshot, type StorageCleanupResult, type StorageCleanupTarget, type StorageItem, type StorageSnapshot, + type StorageSnapshotExtras, } from "../../../shared/types/storage"; +import type { AppResourceUsageSnapshot } from "../../../shared/types"; import { relativeWhen } from "../../lib/format"; -import { COLORS, SANS_FONT, LABEL_STYLE, outlineButton } from "../lanes/laneDesignTokens"; -import { StorageCleanupDialog } from "./storage/StorageCleanupDialog"; +import { appResourcePressureLevel, getAppResourceUsageCoalesced } from "../../lib/resourcePressure"; +import { + COLORS, + SANS_FONT, + LABEL_STYLE, + inlineBadge, + outlineButton, + primaryButton, +} from "../lanes/laneDesignTokens"; +import { SettingsSectionShell } from "./settingsSectionUi"; +import { SmartTooltip } from "../ui/SmartTooltip"; +import { StorageCleanupDialog, type SafeCleanupPlanConfig } from "./storage/StorageCleanupDialog"; import { CATEGORY_META, CATEGORY_ORDER, + DB_COMPACTION_PENDING_HINT, SAFETY_META, baseName, buildCleanupTarget, + buildSafeCleanupPlan, + categoryPolicyChip, cleanableEntries, + daemonMemoryBytes, + dbBreakdownRows, + dbSizeSamples, + dbSizeTrend, + formatApproxBytes, formatBytes, + formatSlowActions, groupLaneItems, + healthChip, + journalEntries, + lastMaintenanceRun, + maintenanceActionLines, + maintenanceHeadline, + safeReclaimableBytes, + sparklinePoints, + type DbSizeSample, + type Trend, } from "./storage/storageView"; +const STORAGE_BRAND = "#38BDF8"; + const PANEL_STYLE: React.CSSProperties = { background: "color-mix(in srgb, var(--color-card) 90%, var(--color-bg) 10%)", border: "1px solid color-mix(in srgb, var(--color-border) 78%, transparent)", @@ -42,12 +83,27 @@ const PANEL_STYLE: React.CSSProperties = { }; type CompressNow = () => Promise<{ filesCompressed: number; savedBytes: number }>; +type RunMaintenanceNow = () => Promise; +type GetRuntimeHealth = () => Promise; function getCompressNow(): CompressNow | undefined { const fn = window.ade?.storage?.compressNow; return typeof fn === "function" ? fn : undefined; } +// runMaintenanceNow / getRuntimeHealth are being added on the main side by a +// sibling workstream. Feature-detect them via a narrow cast so this renderer +// compiles and degrades gracefully whether or not the daemon exposes them yet. +function getRunMaintenanceNow(): RunMaintenanceNow | undefined { + const fn = (window.ade?.storage as { runMaintenanceNow?: unknown } | undefined)?.runMaintenanceNow; + return typeof fn === "function" ? (fn as RunMaintenanceNow) : undefined; +} + +function getRuntimeHealthFn(): GetRuntimeHealth | undefined { + const fn = (window.ade?.app as { getRuntimeHealth?: unknown } | undefined)?.getRuntimeHealth; + return typeof fn === "function" ? (fn as GetRuntimeHealth) : undefined; +} + type CleanupRequest = { title: string; intro: string; targets: StorageCleanupTarget[] }; // --------------------------------------------------------------------------- @@ -77,6 +133,44 @@ function SafetyBadge({ safety }: { safety: StorageItem["safety"] }) { ); } +function PolicyChip({ label }: { label: string }) { + return ( + + {label} + + ); +} + +function TrendArrow({ trend }: { trend: Trend }) { + if (trend === "down") { + return ( + + + + ); + } + if (trend === "up") { + return ( + + + + ); + } + return ( + + – + + ); +} + function BreakdownBar({ categories }: { categories: StorageCategorySnapshot[] }) { const present = categories.filter((category) => category.bytes > 0); const legend = [...present].sort((a, b) => b.bytes - a.bytes); @@ -184,27 +278,39 @@ function Hero({ snapshot, pressureState, refreshing, + reclaimableBytes, onRescan, + onCleanSafely, }: { snapshot: StorageSnapshot; pressureState: DiskPressureSnapshot["state"] | undefined; refreshing: boolean; + reclaimableBytes: number; onRescan: () => void; + onCleanSafely: (() => void) | null; }) { return (
-
- +
+
+ {onCleanSafely && reclaimableBytes > 0 ? ( + + ) : null} + +
Scanned {relativeWhen(snapshot.generatedAt)}
@@ -223,11 +329,15 @@ function Hero({ function CardShell({ categoryId, category, + policyChip, + headerExtra, span, children, }: { categoryId: StorageCategoryId; category: StorageCategorySnapshot; + policyChip?: string; + headerExtra?: React.ReactNode; span?: boolean; children?: React.ReactNode; }) { @@ -241,15 +351,21 @@ function CardShell({ {meta.name}
- - {formatBytes(category.bytes)} - +
+ {headerExtra} + + {formatBytes(category.bytes)} + +
-
+

{meta.description}

- +
+ {policyChip ? : null} + +
{children}
@@ -260,13 +376,20 @@ function ActionButton({ label, icon, onClick, + disabled, }: { label: string; icon: React.ReactNode; onClick: () => void; + disabled?: boolean; }) { return ( - @@ -284,7 +407,7 @@ function ItemRow({ label: string; path?: string; size: string; - detail?: string; + detail?: React.ReactNode; muted?: boolean; action?: React.ReactNode; }) { @@ -341,11 +464,13 @@ function laneDetail(item: StorageItem, archivedAt: string | null | undefined): s function LanesCard({ category, + policyChip, laneIdByKey, archivedAtByKey, onRequestCleanup, }: { category: StorageCategorySnapshot; + policyChip?: string; laneIdByKey: Map; archivedAtByKey: Map; onRequestCleanup: (request: CleanupRequest) => void; @@ -381,7 +506,7 @@ function LanesCard({ if (active.length > 0) summaryParts.push(`${active.length} active`); return ( - + + ) : null} + + + ))} + + + ); +} + +// --------------------------------------------------------------------------- +// Diagnostics strip +// --------------------------------------------------------------------------- + +function DiagnosticTile({ + icon, + label, + value, + sub, + valueColor, +}: { + icon: React.ReactNode; + label: string; + value: React.ReactNode; + sub?: React.ReactNode; + valueColor?: string; +}) { + return ( +
+
+ {icon} + {label} +
+
+ {value} +
+ {sub ?
{sub}
: null} +
+ ); +} + +function Sparkline({ samples }: { samples: DbSizeSample[] }) { + const width = 120; + const height = 30; + const points = sparklinePoints(samples, width, height); + if (points.length < 2) return null; + const path = points.map((p) => `${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" "); + return ( + + + + ); +} + +function DiagnosticsStrip({ + extras, + usage, + usageReady, + runtimeHealth, + runtimeHealthAvailable, +}: { + extras: StorageSnapshotExtras | undefined; + usage: AppResourceUsageSnapshot | null; + usageReady: boolean; + runtimeHealth: RuntimeHealthSnapshot | null; + runtimeHealthAvailable: boolean; +}) { + const samples = React.useMemo(() => dbSizeSamples(extras), [extras]); + const latestDb = samples.length > 0 ? samples[samples.length - 1] : null; + const trend = dbSizeTrend(samples); + const daemonMem = daemonMemoryBytes(usage); + const lastRun = lastMaintenanceRun(extras); + const level = usage ? appResourcePressureLevel(usage) : 0; + const health = healthChip(level); + const healthColor = health.tone === "busy" ? COLORS.danger : health.tone === "elevated" ? COLORS.warning : COLORS.success; + + const notAvailable = Not available yet; + + return ( +
+
+
+

+ Health & diagnostics +

+
+ How the background service is doing on this Mac. +
+
+ {usageReady && usage ? ( + + {health.label} + + ) : null} +
+ +
+ } + label="Database size" + value={ + latestDb ? ( + + {formatBytes(latestDb.bytes)} + {trend ? : null} + + ) : ( + notAvailable + ) + } + sub={samples.length >= 2 ? : latestDb ? "Trend appears after the next cleanup" : undefined} + /> + } + label="Service memory" + value={daemonMem != null ? formatBytes(daemonMem) : notAvailable} + sub={daemonMem != null ? "Resident right now" : undefined} + /> + } + label="Slow responses" + value={ + runtimeHealthAvailable ? ( + 0 ? COLORS.warning : COLORS.textPrimary }}> + {formatSlowActions(runtimeHealth)} + + ) : ( + notAvailable + ) + } + /> + } + label="Last cleanup" + value={lastRun ? {maintenanceHeadline(lastRun)} : notAvailable} + /> +
+
+ ); +} + +// --------------------------------------------------------------------------- +// Maintenance journal +// --------------------------------------------------------------------------- + +function MaintenanceJournal({ extras }: { extras: StorageSnapshotExtras | undefined }) { + const runs = React.useMemo(() => journalEntries(extras, 8), [extras]); + const [expanded, setExpanded] = React.useState(false); + if (runs.length === 0) return null; + + return ( +
+ + + {expanded ? ( +
+ {runs.map((run, index) => ( + + ))} +
+ ) : null} +
+ ); +} + +function JournalRow({ run }: { run: MaintenanceRunReport }) { + const lines = maintenanceActionLines(run).filter((line) => line.detail !== "nothing to do"); + return ( +
+
+ {maintenanceHeadline(run)} +
+ {lines.length > 0 ? ( +
+ {lines.map((line, index) => ( + + {line.label} · {line.detail} + + ))} +
+ ) : null} +
+ ); +} + // --------------------------------------------------------------------------- // Section // --------------------------------------------------------------------------- @@ -460,15 +921,22 @@ export function StorageSection() { const [pressureState, setPressureState] = React.useState(); const [laneIdByKey, setLaneIdByKey] = React.useState>(new Map()); const [archivedAtByKey, setArchivedAtByKey] = React.useState>(new Map()); + const [usage, setUsage] = React.useState(null); + const [usageReady, setUsageReady] = React.useState(false); + const [runtimeHealth, setRuntimeHealth] = React.useState(null); const [loading, setLoading] = React.useState(true); const [refreshing, setRefreshing] = React.useState(false); const [error, setError] = React.useState(null); const [cleanup, setCleanup] = React.useState(null); + const [safeOpen, setSafeOpen] = React.useState(false); const [compressing, setCompressing] = React.useState(false); + const [maintenanceBusy, setMaintenanceBusy] = React.useState(false); const [toast, setToast] = React.useState(null); const toastTimer = React.useRef | null>(null); const compressNow = React.useMemo(() => getCompressNow(), []); + const runMaintenanceNow = React.useMemo(() => getRunMaintenanceNow(), []); + const runtimeHealthFn = React.useMemo(() => getRuntimeHealthFn(), []); const showToast = React.useCallback((message: string) => { setToast(message); @@ -505,12 +973,23 @@ export function StorageSection() { } }, []); + const loadDiagnostics = React.useCallback(async () => { + const [nextUsage, nextHealth] = await Promise.all([ + getAppResourceUsageCoalesced(), + runtimeHealthFn ? runtimeHealthFn().catch(() => null) : Promise.resolve(null), + ]); + setUsage(nextUsage); + setUsageReady(true); + setRuntimeHealth(nextHealth); + }, [runtimeHealthFn]); + React.useEffect(() => { void load(); + void loadDiagnostics(); return () => { if (toastTimer.current) clearTimeout(toastTimer.current); }; - }, [load]); + }, [load, loadDiagnostics]); const runCompress = React.useCallback(async () => { if (!compressNow) return; @@ -526,9 +1005,26 @@ export function StorageSection() { } }, [compressNow, load, showToast]); + const runMaintenanceInline = React.useCallback(async () => { + if (!runMaintenanceNow || maintenanceBusy) return; + setMaintenanceBusy(true); + try { + const report = await runMaintenanceNow(); + const reclaimed = typeof report?.reclaimedBytes === "number" && Number.isFinite(report.reclaimedBytes) ? report.reclaimedBytes : 0; + showToast(reclaimed > 0 ? `Reclaimed ${formatBytes(reclaimed)}` : "Storage is already tidy"); + void load({ force: true, silent: true }); + void loadDiagnostics(); + } catch (err) { + showToast(err instanceof Error ? err.message : "Could not run cleanup"); + } finally { + setMaintenanceBusy(false); + } + }, [runMaintenanceNow, maintenanceBusy, showToast, load, loadDiagnostics]); + const onCleaned = React.useCallback((_result: StorageCleanupResult) => { void load({ force: true, silent: true }); - }, [load]); + void loadDiagnostics(); + }, [load, loadDiagnostics]); const byId = React.useMemo(() => { const map = new Map(); @@ -536,103 +1032,178 @@ export function StorageSection() { return map; }, [snapshot]); + const extras = snapshot?.extras; + const reclaimable = safeReclaimableBytes(extras); + const dbTrend = React.useMemo(() => dbSizeTrend(dbSizeSamples(extras)), [extras]); + + // Assemble the safe-cleanup dialog configuration lazily when opened. + const safeConfig = React.useMemo<{ targets: StorageCleanupTarget[]; plan: SafeCleanupPlanConfig } | null>(() => { + if (!snapshot) return null; + const plan = buildSafeCleanupPlan(snapshot, laneIdByKey); + if (runMaintenanceNow) { + return { + targets: plan.fsTargets, + plan: { + groups: plan.groups, + whatHappens: plan.whatHappens, + estimatedBytes: plan.estimatedBytes, + confirmLabel: "Clean up safely", + runMaintenance: runMaintenanceNow, + onMaintenanceDone: (report) => { + const reclaimed = typeof report?.reclaimedBytes === "number" && Number.isFinite(report.reclaimedBytes) ? report.reclaimedBytes : 0; + showToast(reclaimed > 0 ? `Reclaimed ${formatBytes(reclaimed)}` : "Storage is already tidy"); + }, + }, + }; + } + // Legacy fallback: only filesystem-safe targets are actionable here. + return { + targets: plan.fsTargets, + plan: { + groups: plan.fsGroup ? [plan.fsGroup] : [], + whatHappens: [ + "Remove temporary and rebuildable files ADE recreates on demand.", + "Your chats, projects, active lanes, and backups are never touched.", + ], + estimatedBytes: plan.fsBytes, + confirmLabel: "Clean up safely", + }, + }; + }, [snapshot, laneIdByKey, runMaintenanceNow, showToast]); + + const description = "What ADE keeps on this Mac for this project, and what you can safely clear."; + return ( -
-
-
Settings
-

Storage

-
- What ADE keeps on this Mac for this project, and what you can safely clear. -
-
- - {loading && !snapshot ? ( - - ) : error && !snapshot ? ( -
-
ADE couldn't measure storage right now.
-
{error}
- -
- ) : snapshot ? ( - <> - void load({ force: true })} - /> + +
+ {loading && !snapshot ? ( + + ) : error && !snapshot ? ( +
+
ADE couldn't measure storage right now.
+
{error}
+ +
+ ) : snapshot ? ( + <> + void load({ force: true })} + onCleanSafely={safeConfig ? () => setSafeOpen(true) : null} + /> -
- {CATEGORY_ORDER.map((categoryId) => { - const category = byId.get(categoryId); - if (!category) return null; - if (categoryId === "lanes_worktrees") { +
+ {CATEGORY_ORDER.map((categoryId) => { + const category = byId.get(categoryId); + if (!category) return null; + const policyChip = categoryPolicyChip(extras, categoryId); + if (categoryId === "lanes_worktrees") { + return ( + + ); + } + if (categoryId === "database") { + return ( + void runMaintenanceInline() : null} + maintenanceBusy={maintenanceBusy} + /> + ); + } return ( - void runCompress()} onRequestCleanup={setCleanup} /> ); - } - return ( - void runCompress()} - onRequestCleanup={setCleanup} - /> - ); - })} -
+ })} +
+ + -
- ADE never automatically deletes your chats, project files, active lanes, or backups. + + +
+ ADE never automatically deletes your chats, project files, active lanes, or backups. +
+ + ) : null} + + {toast ? ( +
+ {toast}
- - ) : null} + ) : null} - {toast ? ( -
- {toast} -
- ) : null} + setCleanup(null)} + onCleaned={onCleaned} + /> - setCleanup(null)} - onCleaned={onCleaned} - /> -
+ {safeConfig ? ( + setSafeOpen(false)} + onCleaned={onCleaned} + /> + ) : null} +
+ ); } @@ -641,6 +1212,7 @@ const EMPTY_TARGETS: StorageCleanupTarget[] = []; function CategoryCardBody({ categoryId, category, + policyChip, laneIdByKey, compressNow, compressing, @@ -649,6 +1221,7 @@ function CategoryCardBody({ }: { categoryId: StorageCategoryId; category: StorageCategorySnapshot; + policyChip?: string; laneIdByKey: Map; compressNow: CompressNow | undefined; compressing: boolean; @@ -658,7 +1231,7 @@ function CategoryCardBody({ if (categoryId === "chats_history") { const compressible = category.compressibleBytes ?? 0; return ( - + {compressible > 0 && compressNow ? (
@@ -689,7 +1262,7 @@ function CategoryCardBody({ const cleanable = cleanableEntries(categoryId, category, laneIdByKey); const protectedItems = category.items.filter((item) => item.safety === "protected"); return ( - + {cleanable.length > 0 ? ( sum + entry.item.bytes, 0))}…`} @@ -715,7 +1288,7 @@ function CategoryCardBody({ const cleanable = cleanableEntries(categoryId, category, laneIdByKey); const top = [...category.items].sort((a, b) => b.bytes - a.bytes).slice(0, 3); return ( - + {top.map((item) => ( +
{category.bytes > 0 ? "Review and remove individual items from the proof drawer, where you can see each screenshot and recording." @@ -759,7 +1332,7 @@ function CategoryCardBody({ if (categoryId === "recovery_backups") { return ( - + {category.items.length > 0 ? (
{category.items.map((item) => ( @@ -791,13 +1364,10 @@ function CategoryCardBody({ ); } - // database + // Fallback (should not reach here — database handled above). return ( - -
- - This is your project's live data. ADE protects it automatically. -
+ + ); } diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx index 105631acc..496c0b192 100644 --- a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx +++ b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx @@ -1,6 +1,7 @@ import React from "react"; -import { CircleNotch, Trash, WarningCircle } from "@phosphor-icons/react"; +import { Broom, CircleNotch, Trash, WarningCircle } from "@phosphor-icons/react"; import type { + MaintenanceRunReport, StorageCleanupPreview, StorageCleanupResult, StorageCleanupTarget, @@ -10,8 +11,25 @@ import { SANS_FONT, outlineButton, dangerButton, + primaryButton, } from "../../lanes/laneDesignTokens"; -import { baseName, formatBytes } from "./storageView"; +import { baseName, formatBytes, type SafeCleanupGroup } from "./storageView"; + +/** + * Optional "safe cleanup" plan. When present the dialog runs the broad + * maintenance flow (grouped preview + plain-language "what happens" + a single + * primary confirm) instead of the per-target removal flow. When `runMaintenance` + * is available it drives the daemon doctor; otherwise the dialog falls back to + * the per-target `cleanup` over `targets` (legacy runtimes). + */ +export type SafeCleanupPlanConfig = { + groups: SafeCleanupGroup[]; + whatHappens: string[]; + estimatedBytes: number; + confirmLabel: string; + runMaintenance?: () => Promise; + onMaintenanceDone?: (report: MaintenanceRunReport) => void; +}; type Stage = "loading" | "review" | "removing" | "done" | "error"; @@ -117,6 +135,7 @@ export function StorageCleanupDialog({ title, intro, targets, + plan, onClose, onCleaned, }: { @@ -124,23 +143,44 @@ export function StorageCleanupDialog({ title: string; intro?: string; targets: StorageCleanupTarget[]; + plan?: SafeCleanupPlanConfig; onClose: () => void; onCleaned: (result: StorageCleanupResult) => void; }) { const [stage, setStage] = React.useState("loading"); const [preview, setPreview] = React.useState(null); const [result, setResult] = React.useState(null); + const [report, setReport] = React.useState(null); const [error, setError] = React.useState(null); + // When the maintenance doctor is available we skip the filesystem preview and + // go straight to the itemized plan; the doctor is comprehensive on its own. + const maintenanceMode = Boolean(plan); + const skipPreview = Boolean(plan?.runMaintenance); + + // The preview is initialized once per open. We deliberately do NOT re-run when + // `targets` changes identity: a successful cleanup reloads the parent snapshot, + // which recomputes the (derived) safe-cleanup targets — re-running here would + // reset a "done" dialog back to "review". Read the latest values via a ref. + const initRef = React.useRef({ targets, skipPreview }); + initRef.current = { targets, skipPreview }; + React.useEffect(() => { if (!open) return; let active = true; - setStage("loading"); - setPreview(null); + const { targets: openTargets, skipPreview: openSkip } = initRef.current; setResult(null); + setReport(null); setError(null); + if (openSkip) { + setPreview(null); + setStage("review"); + return; + } + setStage("loading"); + setPreview(null); void window.ade.storage - .cleanupPreview(targets) + .cleanupPreview(openTargets) .then((next) => { if (!active) return; setPreview(next); @@ -154,7 +194,8 @@ export function StorageCleanupDialog({ return () => { active = false; }; - }, [open, targets]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [open]); React.useEffect(() => { if (!open) return; @@ -169,10 +210,28 @@ export function StorageCleanupDialog({ }, [open, stage, onClose]); const confirm = React.useCallback(async () => { - if (!preview || preview.items.length === 0) return; setStage("removing"); setError(null); try { + // Maintenance path: run the daemon doctor and report what it reclaimed. + if (plan?.runMaintenance) { + const nextReport = await plan.runMaintenance(); + const freedBytes = typeof nextReport?.reclaimedBytes === "number" && Number.isFinite(nextReport.reclaimedBytes) + ? nextReport.reclaimedBytes + : 0; + const synthesized: StorageCleanupResult = { removed: [], failed: [], freedBytes }; + setReport(nextReport ?? null); + setResult(synthesized); + setStage("done"); + onCleaned(synthesized); + plan.onMaintenanceDone?.(nextReport); + return; + } + // Fallback / per-target path: remove exactly what was previewed. + if (!preview || preview.items.length === 0) { + setStage("review"); + return; + } const next = await window.ade.storage.cleanup(targets, { preview }); setResult(next); setStage("done"); @@ -181,11 +240,17 @@ export function StorageCleanupDialog({ setError(err instanceof Error ? err.message : String(err)); setStage("error"); } - }, [preview, targets, onCleaned]); + }, [plan, preview, targets, onCleaned]); if (!open) return null; const removableCount = preview?.items.length ?? 0; + // In maintenance mode with the doctor available we always allow confirming + // (the estimate is daemon-provided and the primary action is gated upstream). + const confirmDisabled = stage === "removing" || stage === "loading" || + (skipPreview ? false : removableCount === 0); + const confirmLabel = plan?.confirmLabel + ?? (removableCount > 0 ? `Remove ${removableCount === 1 ? "1 item" : `${removableCount} items`}` : "Remove"); return (
) : null} - {(stage === "review" || stage === "removing") && preview ? ( + {maintenanceMode && plan && (stage === "review" || stage === "removing") ? ( +
+ {plan.groups.length > 0 ? ( +
+ {plan.groups.map((group) => ( +
+
+ {group.heading} +
+ {group.rows.map((row, index) => ( +
+ {row.label} + + {row.size} + +
+ ))} +
+ ))} +
+ ) : null} + + {plan.whatHappens.length > 0 ? ( +
+
+ What happens +
+ {plan.whatHappens.map((line, index) => ( +
+ · + {line} +
+ ))} +
+ ) : null} + +
+ This will free about {formatBytes(plan.estimatedBytes)}. +
+
+ ) : null} + + {!maintenanceMode && (stage === "review" || stage === "removing") && preview ? ( <> {preview.items.length > 0 ? (
@@ -284,8 +434,17 @@ export function StorageCleanupDialog({ {stage === "done" && result ? (
- Freed {formatBytes(result.freedBytes)}. + {result.freedBytes > 0 + ? `Freed ${formatBytes(result.freedBytes)}.` + : maintenanceMode + ? "Storage is already tidy." + : "Nothing needed removing."}
+ {maintenanceMode && report && report.actions.some((a) => a.error) ? ( +
+ Some steps couldn't finish. You can try again later. +
+ ) : null} {result.removed.length > 0 ? (
Removed {result.removed.length === 1 ? "1 item" : `${result.removed.length} items`}. @@ -331,23 +490,25 @@ export function StorageCleanupDialog({ )} diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts new file mode 100644 index 000000000..c26d8b2ac --- /dev/null +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; +import type { + DbBreakdownEntry, + MaintenanceRunReport, + RuntimeHealthSnapshot, + StorageCategorySnapshot, + StorageSnapshotExtras, +} from "../../../../shared/types/storage"; +import type { AppResourceUsageSnapshot } from "../../../../shared/types"; +import { + buildSafeCleanupPlan, + categoryPolicyChip, + daemonMemoryBytes, + dbBreakdownRows, + dbSizeSamples, + dbSizeTrend, + formatApproxBytes, + formatRunClock, + formatSlowActions, + healthChip, + journalEntries, + lastMaintenanceRun, + ledgerLabel, + maintenanceActionLines, + maintenanceHeadline, + safeReclaimableBytes, + sparklinePoints, +} from "./storageView"; + +const MB = 1024 ** 2; + +function run(partial: Partial): MaintenanceRunReport { + return { + startedAt: "2026-07-16T03:12:00.000Z", + finishedAt: "2026-07-16T03:12:05.000Z", + trigger: "daily", + actions: [], + reclaimedBytes: 0, + dbSizeBytes: null, + ...partial, + }; +} + +describe("safeReclaimableBytes / formatApproxBytes", () => { + it("returns the positive daemon estimate, else zero", () => { + expect(safeReclaimableBytes({ safeReclaimableBytes: 5 * MB } as StorageSnapshotExtras)).toBe(5 * MB); + expect(safeReclaimableBytes({ safeReclaimableBytes: 0 } as StorageSnapshotExtras)).toBe(0); + expect(safeReclaimableBytes({ safeReclaimableBytes: -3 } as StorageSnapshotExtras)).toBe(0); + expect(safeReclaimableBytes(undefined)).toBe(0); + }); + + it("prefixes the humanized size with a tilde", () => { + expect(formatApproxBytes(2 * 1024 ** 3)).toBe("~2.0 GB"); + }); +}); + +describe("categoryPolicyChip", () => { + it("reads the chip for a category, ignoring blanks and absence", () => { + const extras = { policyChips: { caches: "Rebuilt on demand", chats_history: " " } } as unknown as StorageSnapshotExtras; + expect(categoryPolicyChip(extras, "caches")).toBe("Rebuilt on demand"); + expect(categoryPolicyChip(extras, "chats_history")).toBeUndefined(); + expect(categoryPolicyChip(extras, "database")).toBeUndefined(); + expect(categoryPolicyChip(undefined, "caches")).toBeUndefined(); + }); +}); + +describe("dbBreakdownRows", () => { + const entries: DbBreakdownEntry[] = [ + { table: "core", label: "Core data", bytes: 8 * MB, category: "core", action: null }, + { table: "automation_ingress_events", label: "Webhook history", bytes: 40 * MB, category: "webhooks", action: "prunable" }, + { table: "operations", label: "Sync bookkeeping", bytes: 20 * MB, category: "sync_bookkeeping", action: "compactable" }, + { table: "pr_snap", label: "Pull request cache", bytes: 4 * MB, category: "pr_cache", action: "compaction_pending" }, + ]; + + it("sorts largest first and maps actions to friendly labels", () => { + const rows = dbBreakdownRows(entries); + expect(rows.map((r) => r.label)).toEqual(["Webhook history", "Sync bookkeeping", "Core data", "Pull request cache"]); + const webhook = rows.find((r) => r.table === "automation_ingress_events")!; + expect(webhook.actionLabel).toBe("Clean up"); + const ops = rows.find((r) => r.table === "operations")!; + expect(ops.actionLabel).toBe("Compact now"); + }); + + it("frames core data as protected with no action", () => { + const core = dbBreakdownRows(entries).find((r) => r.category === "core")!; + expect(core.isProtected).toBe(true); + expect(core.actionLabel).toBeNull(); + }); + + it("marks compaction_pending rows as pending with no action", () => { + const pending = dbBreakdownRows(entries).find((r) => r.category === "pr_cache")!; + expect(pending.isPending).toBe(true); + expect(pending.actionLabel).toBeNull(); + }); + + it("returns nothing for an absent breakdown", () => { + expect(dbBreakdownRows(undefined)).toEqual([]); + }); +}); + +describe("ledgerLabel", () => { + it("maps known ledger ids to plain names", () => { + expect(ledgerLabel("automation.ingress_events")).toBe("Webhook history"); + expect(ledgerLabel("operations.crsql")).toBe("Sync bookkeeping"); + expect(ledgerLabel("review_run_artifacts")).toBe("Review artifacts"); + expect(ledgerLabel("pull_request_snapshots")).toBe("Pull request cache"); + expect(ledgerLabel("transcripts")).toBe("Chat & terminal history"); + }); + + it("title-cases an unknown id without leaking an identifier", () => { + expect(ledgerLabel("some.mystery_thing")).toBe("Mystery thing"); + }); +}); + +describe("maintenanceActionLines", () => { + it("orders by reclaim and humanizes each action", () => { + const report = run({ + actions: [ + { ledgerId: "automation.ingress_events", kind: "prune", itemsAffected: 100, bytesReclaimed: 128 * MB, durationMs: 12 }, + { ledgerId: "operations", kind: "compact", itemsAffected: 0, bytesReclaimed: 0, durationMs: 30 }, + { ledgerId: "review_run_artifacts", kind: "delete", itemsAffected: 3, bytesReclaimed: 0, durationMs: 5, error: "nope" }, + ], + }); + const lines = maintenanceActionLines(report); + expect(lines[0].label).toBe("Webhook history"); + expect(lines[0].detail).toBe("reclaimed 128 MB"); + expect(lines.find((l) => l.label === "Sync bookkeeping")!.detail).toBe("compacted"); + const failed = lines.find((l) => l.label === "Review artifacts")!; + expect(failed.failed).toBe(true); + expect(failed.detail).toBe("couldn't finish"); + }); +}); + +describe("formatRunClock / maintenanceHeadline", () => { + // Build timestamps from LOCAL components so the day boundaries are + // timezone-independent (formatRunClock buckets by the viewer's local day). + const now = new Date(2026, 6, 17, 9, 0, 0); + const todayIso = new Date(2026, 6, 17, 4, 0, 0).toISOString(); + const yesterdayIso = new Date(2026, 6, 16, 3, 12, 0).toISOString(); + + it("labels today and yesterday", () => { + expect(formatRunClock(todayIso, now).startsWith("Today ")).toBe(true); + expect(formatRunClock(yesterdayIso, now).startsWith("Yesterday ")).toBe(true); + }); + + it("builds a headline with the reclaimed amount", () => { + const headline = maintenanceHeadline(run({ finishedAt: yesterdayIso, reclaimedBytes: 481 * MB }), now); + expect(headline).toContain("Yesterday"); + expect(headline).toContain("reclaimed 481 MB"); + }); + + it("says nothing to reclaim when the run freed nothing", () => { + expect(maintenanceHeadline(run({ reclaimedBytes: 0 }), now)).toContain("nothing to reclaim"); + }); +}); + +describe("journalEntries / lastMaintenanceRun", () => { + const older = run({ finishedAt: "2026-07-10T03:00:00.000Z" }); + const newer = run({ finishedAt: "2026-07-16T03:00:00.000Z" }); + const extras = { maintenance: { lastRun: null, journal: [older, newer] } } as unknown as StorageSnapshotExtras; + + it("returns newest first", () => { + expect(journalEntries(extras)[0]).toBe(newer); + }); + + it("prefers the explicit lastRun, else the newest journal entry", () => { + expect(lastMaintenanceRun(extras)).toBe(newer); + const withLast = { maintenance: { lastRun: older, journal: [newer] } } as unknown as StorageSnapshotExtras; + expect(lastMaintenanceRun(withLast)).toBe(older); + expect(lastMaintenanceRun(undefined)).toBeNull(); + }); +}); + +describe("dbSizeSamples / dbSizeTrend / sparklinePoints", () => { + const extras = { + maintenance: { + lastRun: null, + journal: [ + run({ finishedAt: "2026-07-16T03:00:00.000Z", dbSizeBytes: 100 * MB }), + run({ finishedAt: "2026-07-10T03:00:00.000Z", dbSizeBytes: 200 * MB }), + run({ finishedAt: "2026-07-12T03:00:00.000Z", dbSizeBytes: null }), + ], + }, + } as unknown as StorageSnapshotExtras; + + it("extracts positive samples in chronological order", () => { + const samples = dbSizeSamples(extras); + expect(samples.map((s) => s.bytes)).toEqual([200 * MB, 100 * MB]); + }); + + it("detects a downward trend", () => { + expect(dbSizeTrend(dbSizeSamples(extras))).toBe("down"); + expect(dbSizeTrend([{ ts: "x", bytes: 1 }])).toBeNull(); + }); + + it("produces polyline points within the box", () => { + const points = sparklinePoints(dbSizeSamples(extras), 100, 30); + expect(points).toHaveLength(2); + for (const p of points) { + expect(p.x).toBeGreaterThanOrEqual(0); + expect(p.x).toBeLessThanOrEqual(100); + expect(p.y).toBeGreaterThanOrEqual(0); + expect(p.y).toBeLessThanOrEqual(30); + } + }); +}); + +describe("machine health helpers", () => { + it("sums ade-runtime resident memory to bytes, else null", () => { + const usage = { + roleUsage: [ + { role: "ade-runtime", processCount: 1, cpuPercent: 2, memoryMB: 280 }, + { role: "electron-main", processCount: 1, cpuPercent: 5, memoryMB: 500 }, + ], + } as unknown as AppResourceUsageSnapshot; + expect(daemonMemoryBytes(usage)).toBe(280 * MB); + expect(daemonMemoryBytes({} as AppResourceUsageSnapshot)).toBeNull(); + expect(daemonMemoryBytes(null)).toBeNull(); + }); + + it("maps pressure level to a health chip", () => { + expect(healthChip(0).label).toBe("Healthy"); + expect(healthChip(2).label).toBe("Busy"); + expect(healthChip(4).label).toBe("Under load"); + }); + + it("describes slow actions in plain language", () => { + expect(formatSlowActions(null)).toBe("Not available yet"); + expect(formatSlowActions({ slowActions24h: 0 } as RuntimeHealthSnapshot)).toBe("None in the last 24h"); + expect(formatSlowActions({ slowActions24h: 1 } as RuntimeHealthSnapshot)).toBe("1 slow response in 24h"); + expect(formatSlowActions({ slowActions24h: 4 } as RuntimeHealthSnapshot)).toBe("4 slow responses in 24h"); + }); +}); + +describe("buildSafeCleanupPlan", () => { + const categories: StorageCategorySnapshot[] = [ + { + id: "chats_history", + bytes: 900 * MB, + fileCount: 10, + safety: "compressible", + compressibleBytes: 120 * MB, + items: [], + }, + { + id: "caches", + bytes: 300 * MB, + fileCount: 5, + safety: "safe_to_remove", + items: [ + { id: "c1", label: "npm", path: "/proj/.ade/cache/npm", bytes: 300 * MB, fileCount: 5, lastModifiedAt: null, safety: "safe_to_remove" }, + ], + }, + ]; + const extras = { + safeReclaimableBytes: 460 * MB, + dbBreakdown: [ + { table: "automation_ingress_events", label: "Webhook history", bytes: 40 * MB, category: "webhooks", action: "prunable" }, + { table: "core", label: "Core data", bytes: 8 * MB, category: "core", action: null }, + ], + } as unknown as StorageSnapshotExtras; + + it("groups compressible history, database rows, and filesystem targets", () => { + const plan = buildSafeCleanupPlan({ categories, extras }, new Map()); + expect(plan.estimatedBytes).toBe(460 * MB); + expect(plan.groups.map((g) => g.heading)).toEqual([ + "Chats & terminal history", + "Project database", + "Temporary & rebuildable files", + ]); + expect(plan.fsTargets).toHaveLength(1); + expect(plan.fsBytes).toBe(300 * MB); + expect(plan.fsGroup?.rows).toHaveLength(1); + expect(plan.whatHappens.at(-1)).toContain("never touched"); + }); +}); diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.ts index 53021d88c..56278b50b 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.ts @@ -1,11 +1,20 @@ import type { + DbBreakdownCategory, + DbBreakdownEntry, + MaintenanceAction, + MaintenanceRunReport, + RuntimeHealthSnapshot, StorageCategoryId, StorageCategorySnapshot, StorageCleanupTarget, StorageItem, StorageSafety, + StorageSnapshotExtras, } from "../../../../shared/types/storage"; +import type { AppResourceUsageSnapshot } from "../../../../shared/types"; +import type { ResourcePressureLevel } from "../../../lib/resourcePressure"; import { COLORS } from "../../lanes/laneDesignTokens"; +import { formatBytes } from "../../../lib/format"; export { formatBytes } from "../../../lib/format"; /** @@ -174,3 +183,389 @@ export function groupLaneItems(items: StorageItem[]): { } return { active, archived, orphaned }; } + +// =========================================================================== +// Diagnostics & maintenance view-model (overhaul) +// +// These pure helpers turn the optional `StorageSnapshotExtras` (dbBreakdown, +// maintenance journal, policy chips, reclaimable total) and the machine-level +// resource/health snapshots into display-ready shapes. Every helper degrades to +// a sensible "not available" value when its source is missing, so the UI can be +// rendered against an older daemon that never sends `extras`. +// =========================================================================== + +/** Approximate size label, e.g. "~1.2 GB". Used for estimates the doctor will reclaim. */ +export function formatApproxBytes(bytes: number): string { + return `~${formatBytes(bytes)}`; +} + +/** The daemon-provided safe reclaim estimate, clamped to a positive number. */ +export function safeReclaimableBytes(extras: StorageSnapshotExtras | undefined): number { + const n = extras?.safeReclaimableBytes; + return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0; +} + +/** The policy chip string for a category (e.g. "Auto-cleans · 7 days"), or undefined. */ +export function categoryPolicyChip( + extras: StorageSnapshotExtras | undefined, + categoryId: StorageCategoryId, +): string | undefined { + const chip = extras?.policyChips?.[categoryId]; + return typeof chip === "string" && chip.trim().length > 0 ? chip : undefined; +} + +// ---- Project database breakdown ------------------------------------------- + +/** Plain-language framing for each internal database category. No jargon. */ +export const DB_CATEGORY_HINT: Record = { + webhooks: "History of incoming events from your automations.", + sync_bookkeeping: "Records ADE keeps to sync your work across devices.", + review_artifacts: "Saved output from past code reviews.", + pr_cache: "A local copy of pull request details, refetched when needed.", + core: "Your project's live data — chats, lanes, and settings.", +}; + +export type DbBreakdownRow = { + table: string; + label: string; + bytes: number; + size: string; + category: DbBreakdownCategory; + hint: string; + /** Core data is framed as protected and never gets a destructive action. */ + isProtected: boolean; + action: DbBreakdownEntry["action"]; + /** Quiet inline action verb, or null when the row is protected/pending. */ + actionLabel: string | null; + /** Waiting for a safe moment to compact (peers mid-sync); shows a tooltip, no action. */ + isPending: boolean; +}; + +/** Largest-first, display-ready rows for the project database card. */ +export function dbBreakdownRows(entries: DbBreakdownEntry[] | undefined): DbBreakdownRow[] { + if (!entries || entries.length === 0) return []; + return [...entries] + .filter((entry) => entry.bytes > 0 || entry.category === "core") + .sort((a, b) => b.bytes - a.bytes) + .map((entry) => { + const isProtected = entry.category === "core"; + const isPending = entry.action === "compaction_pending"; + const actionLabel = isProtected || isPending + ? null + : entry.action === "prunable" + ? "Clean up" + : entry.action === "compactable" + ? "Compact now" + : null; + return { + table: entry.table, + label: entry.label, + bytes: entry.bytes, + size: formatBytes(entry.bytes), + category: entry.category, + hint: DB_CATEGORY_HINT[entry.category], + isProtected, + action: entry.action, + actionLabel, + isPending, + }; + }); +} + +/** Copy shown in the "waiting to compact" tooltip. Never references sync internals. */ +export const DB_COMPACTION_PENDING_HINT = + "ADE will reclaim this automatically once none of your devices are mid-sync."; + +// ---- Maintenance journal --------------------------------------------------- + +/** + * Best-effort friendly name for a ledger id (e.g. "automation.ingress_events" → + * "Webhook history"). Falls back to a title-cased version of the last segment so + * an unknown ledger entry still reads cleanly and never leaks an identifier. + */ +const LEDGER_LABEL_RULES: Array<{ match: RegExp; label: string }> = [ + { match: /ingress|webhook/i, label: "Webhook history" }, + { match: /operation|tombstone|bookkeep|\bcrr\b|crsql/i, label: "Sync bookkeeping" }, + { match: /review/i, label: "Review artifacts" }, + { match: /pull_?request|pr[_.]?(cache|snapshot)|snapshot/i, label: "Pull request cache" }, + { match: /transcript|chat|terminal|history/i, label: "Chat & terminal history" }, + { match: /tmp|staging|temp/i, label: "Temporary files" }, + { match: /backup|recovery/i, label: "Recovery backups" }, + { match: /cache/i, label: "Caches" }, + { match: /vacuum|freelist|database|\.db|\bdb\b/i, label: "Database file" }, +]; + +export function ledgerLabel(ledgerId: string): string { + for (const rule of LEDGER_LABEL_RULES) { + if (rule.match.test(ledgerId)) return rule.label; + } + const segment = ledgerId.split(/[.:/]/).pop() ?? ledgerId; + const words = segment.replace(/[_-]+/g, " ").trim(); + if (!words) return "Maintenance"; + return words.charAt(0).toUpperCase() + words.slice(1); +} + +/** One humanized line per maintenance action, largest reclaim first. */ +export function maintenanceActionLines( + report: MaintenanceRunReport, +): Array<{ ledgerId: string; label: string; detail: string; failed: boolean }> { + return [...report.actions] + .sort((a, b) => b.bytesReclaimed - a.bytesReclaimed) + .map((action) => ({ + ledgerId: action.ledgerId, + label: ledgerLabel(action.ledgerId), + detail: maintenanceActionDetail(action), + failed: Boolean(action.error), + })); +} + +function maintenanceActionDetail(action: MaintenanceAction): string { + if (action.error) return "couldn't finish"; + if (action.skippedReason) return "nothing to do"; + if (action.bytesReclaimed > 0) return `reclaimed ${formatBytes(action.bytesReclaimed)}`; + if (action.itemsAffected > 0) { + return `${action.itemsAffected} ${action.itemsAffected === 1 ? "item" : "items"}`; + } + return actionVerbPast(action.kind); +} + +function actionVerbPast(kind: MaintenanceAction["kind"]): string { + switch (kind) { + case "compress": + return "compressed"; + case "compact": + return "compacted"; + case "prune": + case "delete": + return "cleaned up"; + case "vacuum": + return "reclaimed space"; + case "checkpoint": + return "tidied up"; + default: + return "done"; + } +} + +/** + * A friendly clock label for a run timestamp: "Today 03:12", "Yesterday 03:12", + * a weekday within a week, else "Jul 12 03:12". `now` is injectable for tests. + */ +export function formatRunClock(iso: string, now: Date = new Date()): string { + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return ""; + const time = date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + const startOfDay = (d: Date) => new Date(d.getFullYear(), d.getMonth(), d.getDate()).getTime(); + const dayDelta = Math.round((startOfDay(now) - startOfDay(date)) / 86_400_000); + if (dayDelta <= 0) return `Today ${time}`; + if (dayDelta === 1) return `Yesterday ${time}`; + if (dayDelta < 7) return `${date.toLocaleDateString([], { weekday: "short" })} ${time}`; + return `${date.toLocaleDateString([], { month: "short", day: "numeric" })} ${time}`; +} + +/** Headline for a maintenance run, e.g. "Yesterday 03:12 · reclaimed 481 MB". */ +export function maintenanceHeadline(report: MaintenanceRunReport, now?: Date): string { + const when = formatRunClock(report.finishedAt || report.startedAt, now); + const reclaimed = report.reclaimedBytes > 0 + ? `reclaimed ${formatBytes(report.reclaimedBytes)}` + : "nothing to reclaim"; + return when ? `${when} · ${reclaimed}` : reclaimed; +} + +/** Recent runs, newest first, capped for display. */ +export function journalEntries( + extras: StorageSnapshotExtras | undefined, + limit = 8, +): MaintenanceRunReport[] { + const journal = extras?.maintenance?.journal ?? []; + return [...journal] + .sort((a, b) => runTime(b) - runTime(a)) + .slice(0, Math.max(0, limit)); +} + +/** The most recent run (prefer the explicit lastRun, else the newest journal entry). */ +export function lastMaintenanceRun( + extras: StorageSnapshotExtras | undefined, +): MaintenanceRunReport | null { + return extras?.maintenance?.lastRun ?? journalEntries(extras, 1)[0] ?? null; +} + +function runTime(report: MaintenanceRunReport): number { + const ts = Date.parse(report.finishedAt || report.startedAt); + return Number.isNaN(ts) ? 0 : ts; +} + +// ---- Database-size trend & sparkline -------------------------------------- + +export type DbSizeSample = { ts: string; bytes: number }; + +/** Database-size samples from doctor runs, chronological, positive-valued only. */ +export function dbSizeSamples(extras: StorageSnapshotExtras | undefined): DbSizeSample[] { + const journal = extras?.maintenance?.journal ?? []; + const out: DbSizeSample[] = []; + for (const run of journal) { + const bytes = run.dbSizeBytes; + const ts = run.finishedAt || run.startedAt; + if (typeof bytes === "number" && Number.isFinite(bytes) && bytes > 0 && ts && !Number.isNaN(Date.parse(ts))) { + out.push({ ts, bytes }); + } + } + out.sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts)); + return out; +} + +export type Trend = "down" | "up" | "flat"; + +/** Trend of the last two database samples, or null when there aren't two. */ +export function dbSizeTrend(samples: DbSizeSample[]): Trend | null { + if (samples.length < 2) return null; + const prev = samples[samples.length - 2].bytes; + const last = samples[samples.length - 1].bytes; + if (prev <= 0) return null; + const ratio = last / prev; + if (ratio < 0.99) return "down"; + if (ratio > 1.01) return "up"; + return "flat"; +} + +/** Normalized polyline points (0..1 in both axes, y flipped for SVG) for a sparkline. */ +export function sparklinePoints( + samples: DbSizeSample[], + width: number, + height: number, +): Array<{ x: number; y: number }> { + if (samples.length === 0) return []; + if (samples.length === 1) { + return [{ x: 0, y: height / 2 }, { x: width, y: height / 2 }]; + } + const values = samples.map((s) => s.bytes); + const min = Math.min(...values); + const max = Math.max(...values); + const span = max - min || 1; + const step = width / (samples.length - 1); + return samples.map((sample, index) => ({ + x: index * step, + y: height - ((sample.bytes - min) / span) * height, + })); +} + +// ---- Machine health -------------------------------------------------------- + +/** Resident memory of the ADE background service (ade-runtime role), in bytes, or null. */ +export function daemonMemoryBytes(usage: AppResourceUsageSnapshot | null): number | null { + const roleUsage = usage?.roleUsage; + if (!roleUsage) return null; + let mb = 0; + let found = false; + for (const entry of roleUsage) { + if (entry.role !== "ade-runtime") continue; + if (typeof entry.memoryMB === "number" && Number.isFinite(entry.memoryMB)) { + mb += entry.memoryMB; + found = true; + } + } + return found ? mb * 1024 * 1024 : null; +} + +export type HealthTone = "good" | "elevated" | "busy"; + +/** Overall health chip derived from the resource-pressure level (0..4). */ +export function healthChip(level: ResourcePressureLevel): { label: string; tone: HealthTone } { + if (level >= 3) return { label: "Under load", tone: "busy" }; + if (level === 2) return { label: "Busy", tone: "elevated" }; + return { label: "Healthy", tone: "good" }; +} + +/** Plain sentence for the slow-actions tile. */ +export function formatSlowActions(health: RuntimeHealthSnapshot | null): string { + if (!health) return "Not available yet"; + const n = health.slowActions24h; + if (!Number.isFinite(n) || n <= 0) return "None in the last 24h"; + return `${n} slow ${n === 1 ? "response" : "responses"} in 24h`; +} + +// ---- Safe-cleanup plan ----------------------------------------------------- + +export type SafeCleanupGroup = { + heading: string; + rows: Array<{ label: string; size: string }>; +}; + +export type SafeCleanupPlan = { + /** Filesystem targets used only for the legacy fallback (cleanup-by-target). */ + fsTargets: StorageCleanupTarget[]; + /** Sum of the filesystem-safe targets' sizes (the legacy fallback reclaim). */ + fsBytes: number; + /** Grouped, itemized preview of what the safe cleanup will reclaim. */ + groups: SafeCleanupGroup[]; + /** Just the filesystem group, for the legacy fallback preview. */ + fsGroup: SafeCleanupGroup | null; + /** Plain-language "what happens" lines. */ + whatHappens: string[]; + /** Approximate total the doctor will reclaim. */ + estimatedBytes: number; +}; + +/** + * Assemble the itemized "Clean up safely" plan from the snapshot's extras. Only + * meaningful when `extras.safeReclaimableBytes` is positive (the primary action + * is otherwise hidden). Groups the reclaimable database rows and compressible + * history, and collects the filesystem-safe targets for the legacy fallback path. + */ +export function buildSafeCleanupPlan( + snapshot: { categories: StorageCategorySnapshot[]; extras?: StorageSnapshotExtras }, + laneIdByKey: Map, +): SafeCleanupPlan { + const extras = snapshot.extras; + const estimatedBytes = safeReclaimableBytes(extras); + + const groups: SafeCleanupGroup[] = []; + const whatHappens: string[] = []; + + // Compressible chat & terminal history. + const chats = snapshot.categories.find((c) => c.id === "chats_history"); + const compressible = chats?.compressibleBytes ?? 0; + if (compressible > 0) { + groups.push({ + heading: "Chats & terminal history", + rows: [{ label: "Older history, compressed in place", size: formatBytes(compressible) }], + }); + whatHappens.push("Compress older chat and terminal history — nothing is lost."); + } + + // Reclaimable database rows (prunable/compactable). + const dbRows = (extras?.dbBreakdown ?? []) + .filter((entry) => entry.action === "prunable" || entry.action === "compactable") + .sort((a, b) => b.bytes - a.bytes) + .map((entry) => ({ label: entry.label, size: formatBytes(entry.bytes) })); + if (dbRows.length > 0) { + groups.push({ heading: "Project database", rows: dbRows }); + whatHappens.push("Clean up data ADE keeps but no longer needs."); + } + + // Filesystem-safe targets (caches + build/release staging) for the fallback path. + const fsTargets: StorageCleanupTarget[] = []; + const fsRows: Array<{ label: string; size: string }> = []; + let fsBytes = 0; + for (const category of snapshot.categories) { + if (category.id !== "caches" && category.id !== "build_release") continue; + for (const entry of cleanableEntries(category.id, category, laneIdByKey)) { + fsTargets.push(entry.target); + fsRows.push({ label: entry.item.label, size: formatBytes(entry.item.bytes) }); + fsBytes += entry.item.bytes; + } + } + const fsGroup: SafeCleanupGroup | null = fsRows.length > 0 + ? { heading: "Temporary & rebuildable files", rows: fsRows } + : null; + if (fsGroup) { + groups.push(fsGroup); + whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); + } else if (estimatedBytes > 0) { + whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); + } + + whatHappens.push("Your chats, projects, active lanes, and backups are never touched."); + + return { fsTargets, fsBytes, groups, fsGroup, whatHappens, estimatedBytes }; +} diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts index 0889c2a31..cc52081ea 100644 --- a/apps/desktop/src/shared/ipc.ts +++ b/apps/desktop/src/shared/ipc.ts @@ -2,9 +2,11 @@ export const IPC = { appPing: "ade.app.ping", appGetInfo: "ade.app.getInfo", appGetResourceUsage: "ade.app.getResourceUsage", + appGetRuntimeHealth: "ade.app.getRuntimeHealth", storageGetPressure: "ade.storage.getPressure", storageGetSnapshot: "ade.storage.getSnapshot", storageCompressNow: "ade.storage.compressNow", + storageRunMaintenanceNow: "ade.storage.runMaintenanceNow", storageCleanupPreview: "ade.storage.cleanupPreview", storageCleanup: "ade.storage.cleanup", appGetLatestRelease: "ade.app.getLatestRelease", diff --git a/apps/desktop/src/shared/types/storage.ts b/apps/desktop/src/shared/types/storage.ts index 603caba35..e6502dc27 100644 --- a/apps/desktop/src/shared/types/storage.ts +++ b/apps/desktop/src/shared/types/storage.ts @@ -67,6 +67,9 @@ export type StorageSnapshot = { categories: StorageCategorySnapshot[]; scanDurationMs: number; truncated: boolean; + // Optional so pre-overhaul snapshots (and the daemon-backed fallback + // instance) remain valid; renderer must treat absence as "not available". + extras?: StorageSnapshotExtras; }; export type StorageCleanupTarget = @@ -91,3 +94,84 @@ export type StorageCleanupResult = { failed: Array<{ path: string; reason: string }>; freedBytes: number; }; + +export type StoragePolicyClass = "user_data" | "derived" | "operational"; + +export type StorageLedgerPolicy = { + maxAgeDays?: number; + maxRows?: number; + maxBytes?: number; + compressAfterDays?: number; + keepLatest?: number; +}; + +export type StorageLedgerEntry = { + id: string; + kind: "table" | "directory" | "file_family"; + description: string; + policyClass: StoragePolicyClass; + policy: StorageLedgerPolicy; + enforcement: "write_time" | "doctor" | "both" | "manual"; +}; + +export type MaintenanceActionKind = + | "prune" + | "compress" + | "delete" + | "vacuum" + | "compact" + | "checkpoint"; + +export type MaintenanceAction = { + ledgerId: string; + kind: MaintenanceActionKind; + itemsAffected: number; + bytesReclaimed: number; + durationMs: number; + skippedReason?: string | null; + error?: string | null; +}; + +export type MaintenanceTrigger = "daily" | "post_boot" | "manual" | "post_migration"; + +export type MaintenanceRunReport = { + startedAt: string; + finishedAt: string; + trigger: MaintenanceTrigger; + actions: MaintenanceAction[]; + reclaimedBytes: number; + dbSizeBytes: number | null; +}; + +export type DbBreakdownCategory = + | "webhooks" + | "sync_bookkeeping" + | "review_artifacts" + | "pr_cache" + | "core"; + +export type DbBreakdownEntry = { + table: string; + label: string; + bytes: number; + category: DbBreakdownCategory; + action: "prunable" | "compactable" | "compaction_pending" | null; +}; + +export type StorageMaintenanceInfo = { + lastRun: MaintenanceRunReport | null; + journal: MaintenanceRunReport[]; +}; + +export type StorageSnapshotExtras = { + dbBreakdown: DbBreakdownEntry[]; + maintenance: StorageMaintenanceInfo; + safeReclaimableBytes: number; + policyChips: Partial>; +}; + +export type RuntimeHealthSnapshot = { + slowActions24h: number; + slowActionP95Ms: number | null; + sampledAt: string; +}; diff --git a/docs/logging.md b/docs/logging.md index a51ae233e..e1c48fab2 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -83,6 +83,8 @@ Persisted `usage_events` are the preferred source for meaningful user mutations. Daily usage summaries report coarse totals and only the top coarse provider and model family. They never report provider account IDs, exact model strings, prompt content, or per-session content. +The storage doctor emits one `ade_feature_used` per completed maintenance run at the daemon boundary (`storageInsightsService`), with `feature: "storage_doctor"`, `action: "maintenance_run"`, a coarse `outcome` (`completed`, `partial`, or `failed`), and the numeric aggregates `bytes_freed` and `files_compressed`. It carries no paths, table names, or per-item detail. A per-project local dedupe key (`storage_doctor_run:`) with a 20 h minimum interval collapses the daily run and any manual "Clean up now" into a single accepted event, so worst-case volume is well under 2 accepted events per project per day — inside the shared 200-event ceiling and the `ade_feature_used` per-day cap. The run also writes the local `storage.maintenance_completed` jsonl line (with `storage.maintenance_step_failed` per failed step); those operational lines are never forwarded to PostHog. + ### Native iOS Native UI analytics lives in `apps/ios/ADE/Services/ProductAnalytics.swift`. It uses a separate anonymous installation identity and separate `ade_mobile_*` event namespace so phone engagement cannot inflate desktop activation or retention. From 467aa855fd86d61d17807380a4999352e81407c8 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:50:55 -0400 Subject: [PATCH 02/10] =?UTF-8?q?fix(storage):=20apply=20/quality=20synthe?= =?UTF-8?q?sis=20=E2=80=94=20backup=20keep-newest,=20honest=20copy,=20resi?= =?UTF-8?q?lient=20reclaim,=20structural=20extractions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - doctor never reaps the newest recovery backup (keepLatest:1 honored); copy no longer claims backups are never touched - sync-bookkeeping breakdown defaults to "waiting" until a compact run proves peer-free; tooltip no longer promises automatic reclaim - one-time ingress reclaim is non-fatal on error (disk-full machines can still open projects); dispatched events exempt from the count cap so replays can't re-run automations under webhook storms - vacuumIfFragmented prefers bounded incremental passes once auto_vacuum is active; maintenance journal writes atomically (tmp+rename) - .ade/tmp staging mapped in the renderer cleanup targets; scan skips symlinked staging like the reaper does - retention windows hoisted to dbMaintenanceApi (single source of truth); db-breakdown + journal helpers and diagnostics/journal components extracted to their own modules (1k-line rule) Co-Authored-By: Claude Fable 5 --- .../automations/automationService.test.ts | 64 +++- .../services/automations/automationService.ts | 30 +- .../main/services/state/dbMaintenanceApi.ts | 14 + .../state/kvDb.rebuildRecovery.test.ts | 32 +- apps/desktop/src/main/services/state/kvDb.ts | 49 ++- .../services/storage/storageDbBreakdown.ts | 75 +++++ .../storage/storageInsightsService.test.ts | 29 +- .../storage/storageInsightsService.ts | 138 ++------- .../main/services/storage/storageLedger.ts | 17 +- .../storage/storageMaintenanceJournal.ts | 59 ++++ apps/desktop/src/preload/preload.ts | 2 +- .../components/settings/StorageSection.tsx | 292 +----------------- .../settings/storage/StorageDiagnostics.tsx | 201 ++++++++++++ .../storage/StorageMaintenanceJournal.tsx | 80 +++++ .../settings/storage/storageUiConstants.ts | 14 + .../settings/storage/storageView.test.ts | 51 ++- .../settings/storage/storageView.ts | 56 ++-- 17 files changed, 739 insertions(+), 464 deletions(-) create mode 100644 apps/desktop/src/main/services/storage/storageDbBreakdown.ts create mode 100644 apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts create mode 100644 apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx create mode 100644 apps/desktop/src/renderer/components/settings/storage/StorageMaintenanceJournal.tsx create mode 100644 apps/desktop/src/renderer/components/settings/storage/storageUiConstants.ts diff --git a/apps/desktop/src/main/services/automations/automationService.test.ts b/apps/desktop/src/main/services/automations/automationService.test.ts index 24ac97964..e91d3b27b 100644 --- a/apps/desktop/src/main/services/automations/automationService.test.ts +++ b/apps/desktop/src/main/services/automations/automationService.test.ts @@ -2525,6 +2525,33 @@ describe("automation ingress storage bounds", () => { expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where raw_payload_json is not null", ))[0]?.count).toBe(0); + + // Dispatched rows are exempt from the count cap; an equally-old ignored + // row beyond the newest 2,000 is not. Both are within the 7-day window so + // the age prune leaves them; only the count cap acts. + const fiveDaysAgo = new Date(Date.now() - 5 * 24 * 60 * 60 * 1_000).toISOString(); + raw.run( + `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) + values ('keep-dispatched', 'proj', 'github-relay', 'keep-dispatched', '[]', 'github.issue_opened', 'dispatched', null, ?)`, + [fiveDaysAgo], + ); + raw.run( + `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) + values ('drop-ignored', 'proj', 'github-relay', 'drop-ignored', '[]', 'github.issue_opened', 'ignored', null, ?)`, + [fiveDaysAgo], + ); + // A fresh dispatch runs prune-on-insert. + await service.dispatchIngressTrigger({ + source: "github-relay", + eventKey: "count-cap-trigger", + triggerType: "github.issue_opened", + }); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where event_key = 'keep-dispatched'", + ))[0]?.count).toBe(1); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where event_key = 'drop-ignored'", + ))[0]?.count).toBe(0); } finally { service.dispose(); } @@ -2567,9 +2594,11 @@ describe("automation ingress storage bounds", () => { expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where raw_payload_json is not null", ))[0]?.count).toBe(0); + // 2,000 newest non-dispatched rows + the 3 dispatched rows (exempt from + // the count cap) survive; the 100 aged-out rows are pruned by age. expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where project_id = 'proj'", - ))[0]?.count).toBe(2_000); + ))[0]?.count).toBe(2_003); expect(mapExecRows(raw.exec("select id from review_run_artifacts order by id"))).toEqual([{ id: "new-review" }]); expect(mapExecRows(raw.exec("select pr_id from pull_request_snapshots order by pr_id"))).toEqual([{ pr_id: "new-pr" }]); expect(info).toHaveBeenCalledWith("automations.ingress_payload_reclaim", { @@ -2584,9 +2613,40 @@ describe("automation ingress storage bounds", () => { expect(info).toHaveBeenCalledTimes(logCount); expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where project_id = 'proj'", - ))[0]?.count).toBe(2_000); + ))[0]?.count).toBe(2_003); } finally { first.dispose(); } }); + + it("survives a failure while reclaiming legacy ingress payloads", () => { + const { db, raw } = createInMemoryAdeDb(); + raw.run( + `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) + values ('legacy-1', 'proj', 'github-relay', 'legacy-key-1', '[]', 'github.issue_opened', 'ignored', ?, ?)`, + [JSON.stringify({ body: "legacy" }), new Date().toISOString()], + ); + const warn = vi.fn(); + const logger = { ...createLogger(), warn } as any; + // Force the reclaim UPDATE to throw; every other statement runs normally. + const failingDb: AdeDb = { + ...db, + run: (sql: string, params?: SqlValue[]) => { + if (typeof sql === "string" && sql.includes("set raw_payload_json = null")) { + throw new Error("reclaim boom"); + } + return db.run(sql, params); + }, + }; + const service = createIngressService(failingDb, logger); + service.dispose(); + expect(warn).toHaveBeenCalledWith( + "automations.ingress_payload_reclaim_failed", + expect.objectContaining({ error: expect.stringContaining("reclaim boom") }), + ); + // Construction still succeeded and the legacy row is left in place to retry. + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where raw_payload_json is not null", + ))[0]?.count).toBe(1); + }); }); diff --git a/apps/desktop/src/main/services/automations/automationService.ts b/apps/desktop/src/main/services/automations/automationService.ts index 16c0fa595..a9781623a 100644 --- a/apps/desktop/src/main/services/automations/automationService.ts +++ b/apps/desktop/src/main/services/automations/automationService.ts @@ -35,6 +35,12 @@ import type { import { triggerDeliveryKeyForType } from "../../../shared/types"; import type { Logger } from "../logging/logger"; import type { AdeDb, SqlValue } from "../state/kvDb"; +import { + INGRESS_EVENT_MAX_ROWS_PER_PROJECT, + INGRESS_EVENT_RETENTION_MS, + PR_SNAPSHOT_RETENTION_DAYS, + REVIEW_ARTIFACT_RETENTION_DAYS, +} from "../state/dbMaintenanceApi"; import type { createLaneService } from "../lanes/laneService"; import type { createProjectConfigService } from "../config/projectConfigService"; import type { createConflictService } from "../conflicts/conflictService"; @@ -357,11 +363,11 @@ type AutomationIngressEventRow = { received_at: string; }; -const INGRESS_EVENT_RETENTION_MS = 7 * 24 * 60 * 60 * 1_000; -const INGRESS_EVENT_MAX_ROWS_PER_PROJECT = 2_000; +// Retention/count bounds live in `state/dbMaintenanceApi` so the ingress writer, +// the kvDb maintenance hooks, and the storage ledger enforce identical policy. const INGRESS_PAYLOAD_RECLAIM_CHUNK_ROWS = 2_000; -const REVIEW_ARTIFACT_RETENTION_MS = 30 * 24 * 60 * 60 * 1_000; -const PR_SNAPSHOT_RETENTION_MS = 60 * 24 * 60 * 60 * 1_000; +const REVIEW_ARTIFACT_RETENTION_MS = REVIEW_ARTIFACT_RETENTION_DAYS * 24 * 60 * 60 * 1_000; +const PR_SNAPSHOT_RETENTION_MS = PR_SNAPSHOT_RETENTION_DAYS * 24 * 60 * 60 * 1_000; type AutomationPendingPublishRow = { id: string; @@ -1137,12 +1143,15 @@ export function createAutomationService({ "delete from automation_ingress_events where project_id = ? and received_at < ?", [targetProjectId, cutoff], ); + // Dispatched events are exempt from the count cap (they may still be read + // by run detail / the UI); they remain subject to the age prune above, so + // they still expire at the retention horizon. db.run( `delete from automation_ingress_events where rowid in ( select rowid from automation_ingress_events - where project_id = ? + where project_id = ? and status != 'dispatched' order by received_at desc, rowid desc limit -1 offset ${INGRESS_EVENT_MAX_ROWS_PER_PROJECT} )`, @@ -1363,7 +1372,16 @@ export function createAutomationService({ // delete completed before the crash, the missing-lane path records success. db.run("update automation_scheduled_cleanups set status = 'scheduled' where status = 'executing' and project_id = ?", [projectId]); - reclaimLegacyIngressPayloads(); + // A one-time reclaim failure must never block service construction: the + // service still functions with the legacy payloads in place (they are only + // dead weight), and the next boot retries the reclaim. + try { + reclaimLegacyIngressPayloads(); + } catch (error) { + logger.warn("automations.ingress_payload_reclaim_failed", { + error: error instanceof Error ? error.message : String(error), + }); + } }; diff --git a/apps/desktop/src/main/services/state/dbMaintenanceApi.ts b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts index 16d4074e9..bcd356949 100644 --- a/apps/desktop/src/main/services/state/dbMaintenanceApi.ts +++ b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts @@ -3,6 +3,20 @@ // by storageInsightsService via optional chaining so the doctor degrades // gracefully on handles that predate the implementation. +// Single source of truth for the retention/count bounds enforced across the +// automation ingress writer, the kvDb maintenance hooks, and the storage +// ledger. Hoisted here so a policy change updates every enforcement site and +// the Settings copy in lockstep. +const DAY_MS = 24 * 60 * 60 * 1_000; +/** Automation ingress events older than this are pruned (write-time + doctor). */ +export const INGRESS_EVENT_RETENTION_MS = 7 * DAY_MS; +/** Newest ingress rows kept per project regardless of age (count cap). */ +export const INGRESS_EVENT_MAX_ROWS_PER_PROJECT = 2_000; +/** Review artifacts older than this (in days) are deleted. */ +export const REVIEW_ARTIFACT_RETENTION_DAYS = 30; +/** PR snapshots not updated within this many days are deleted. */ +export const PR_SNAPSHOT_RETENTION_DAYS = 60; + export type DbMaintenanceResult = { itemsAffected: number; bytesReclaimed: number; diff --git a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts index d0f64f5ef..3d8a84142 100644 --- a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts +++ b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts @@ -407,7 +407,8 @@ describe("kvDb storage maintenance", () => { it("reclaims a fragmented file, activates incremental auto-vacuum, and skips a healthy file", async () => { const dbPath = makeDbPath(); - const db = await openKvDb(dbPath, createLogger()); + const logs: LogEntry[] = []; + const db = await openKvDb(dbPath, createLogger(logs)); closeLater(db); db.run("create table maintenance_fragmentation(id integer primary key, payload text not null)"); db.run("begin immediate"); @@ -434,6 +435,35 @@ describe("kvDb storage maintenance", () => { expect(fs.statSync(dbPath).size).toBeLessThan(beforeBytes); expect(db.get<{ auto_vacuum: number }>("pragma auto_vacuum")?.auto_vacuum).toBe(2); + // The first fragmented pass ran a one-time full VACUUM. + const vacuumModes = () => logs + .filter((entry) => entry.event === "db.maintenance_vacuum") + .map((entry) => entry.fields.mode); + expect(vacuumModes().at(-1)).toBe("full"); + + // Re-fragment above the threshold. With incremental auto-vacuum now active, + // a second fragmented pass must use the bounded incremental path, never a + // second blocking full VACUUM. + db.run("begin immediate"); + for (let index = 0; index < 2_000; index += 1) { + db.run( + "insert into maintenance_fragmentation(id, payload) values (?, ?)", + [index, `${index}-${"z".repeat(4_000)}`], + ); + } + db.run("commit"); + db.flushNow(); + db.run("delete from maintenance_fragmentation"); + db.flushNow(); + const refragPages = Number(db.get<{ freelist_count: number }>("pragma freelist_count")?.freelist_count ?? 0); + const refragTotal = Number(db.get<{ page_count: number }>("pragma page_count")?.page_count ?? 0); + expect(refragPages / refragTotal).toBeGreaterThan(0.2); + const secondFragmented = db.maintenance?.vacuumIfFragmented(0.2); + expect(secondFragmented?.skippedReason).toBeNull(); + expect(secondFragmented?.itemsAffected).toBeGreaterThan(0); + expect(vacuumModes().at(-1)).toBe("incremental"); + expect(vacuumModes().filter((mode) => mode === "full")).toHaveLength(1); + db.run("begin immediate"); for (let index = 0; index < 256; index += 1) { db.run( diff --git a/apps/desktop/src/main/services/state/kvDb.ts b/apps/desktop/src/main/services/state/kvDb.ts index f356b61b5..2f89facb4 100644 --- a/apps/desktop/src/main/services/state/kvDb.ts +++ b/apps/desktop/src/main/services/state/kvDb.ts @@ -9,7 +9,14 @@ import type { Logger } from "../logging/logger"; import { safeJsonParse } from "../shared/utils"; import { isNoSpaceError, readVolumeSpace } from "../storage/volume"; import { resolveCrsqliteExtensionPath } from "./crsqliteExtension"; -import type { DbMaintenanceApi, DbMaintenanceResult } from "./dbMaintenanceApi"; +import { + INGRESS_EVENT_MAX_ROWS_PER_PROJECT, + INGRESS_EVENT_RETENTION_MS, + PR_SNAPSHOT_RETENTION_DAYS, + REVIEW_ARTIFACT_RETENTION_DAYS, + type DbMaintenanceApi, + type DbMaintenanceResult, +} from "./dbMaintenanceApi"; import type { ApplyRemoteChangesResult, CrsqlChangeRow, SyncScalar } from "../../../shared/types/sync"; type DatabaseSyncConstructor = new (dbPath: string, options?: { allowExtension?: boolean; readOnly?: boolean }) => DatabaseSyncType; @@ -3690,22 +3697,46 @@ export async function openKvDb( const freePagesBefore = Number(getRow<{ freelist_count: number }>(db, "pragma freelist_count")?.freelist_count ?? 0); const fragmentation = pageCount > 0 ? freePagesBefore / pageCount : 0; const autoVacuum = Number(getRow<{ auto_vacuum: number }>(db, "pragma auto_vacuum")?.auto_vacuum ?? 0); + const AUTO_VACUUM_INCREMENTAL = 2; + // Bounded per-call incremental budget: at most 25 chunks * 2,000 pages = + // 50k pages reclaimed without ever running a blocking full VACUUM again. + const INCREMENTAL_VACUUM_CHUNK_PAGES = 2_000; + const INCREMENTAL_VACUUM_MAX_ITERATIONS = 25; let itemsAffected = 0; let skippedReason: DbMaintenanceResult["skippedReason"] = null; let mode: "full" | "incremental" | "none" = "none"; - if (pageCount > 0 && fragmentation >= threshold) { - // SQLite applies a NONE -> INCREMENTAL auto-vacuum transition when the - // following VACUUM rebuilds the file, so the pragma must come first. + const readFreePages = (): number => + Number(getRow<{ freelist_count: number }>(db, "pragma freelist_count")?.freelist_count ?? 0); + const readFragmentation = (freePages: number): number => { + const pages = Number(getRow<{ page_count: number }>(db, "pragma page_count")?.page_count ?? 0); + return pages > 0 ? freePages / pages : 0; + }; + + if (autoVacuum === AUTO_VACUUM_INCREMENTAL) { + // Incremental auto-vacuum is already active: never run a blocking full + // VACUUM again. Reclaim the freelist in bounded chunks until it drops + // below the threshold fraction or the per-call budget is exhausted. + if (freePagesBefore > 0) { + mode = "incremental"; + let freeNow = freePagesBefore; + for (let iteration = 0; iteration < INCREMENTAL_VACUUM_MAX_ITERATIONS && freeNow > 0; iteration += 1) { + db.exec(`PRAGMA incremental_vacuum(${INCREMENTAL_VACUUM_CHUNK_PAGES})`); + freeNow = readFreePages(); + if (readFragmentation(freeNow) < threshold) break; + } + itemsAffected = Math.max(0, freePagesBefore - freeNow); + } else { + skippedReason = "below_threshold"; + } + } else if (pageCount > 0 && fragmentation >= threshold) { + // Not yet incremental: a one-time full VACUUM rebuilds the file and, via + // the preceding pragma, activates incremental auto-vacuum so every later + // sweep uses the bounded path above instead of another full VACUUM. db.exec("PRAGMA auto_vacuum = INCREMENTAL"); db.exec("VACUUM"); itemsAffected = freePagesBefore; mode = "full"; - } else if (autoVacuum === 2 && freePagesBefore > 0) { - db.exec("PRAGMA incremental_vacuum(2000)"); - const freePagesAfter = Number(getRow<{ freelist_count: number }>(db, "pragma freelist_count")?.freelist_count ?? 0); - itemsAffected = Math.max(0, freePagesBefore - freePagesAfter); - mode = "incremental"; } else { skippedReason = "below_threshold"; } diff --git a/apps/desktop/src/main/services/storage/storageDbBreakdown.ts b/apps/desktop/src/main/services/storage/storageDbBreakdown.ts new file mode 100644 index 000000000..26f4c73d5 --- /dev/null +++ b/apps/desktop/src/main/services/storage/storageDbBreakdown.ts @@ -0,0 +1,75 @@ +// Pure helpers that turn raw dbstat rows into the coarse "project database" +// breakdown shown in Settings, plus the derivation of the sync-bookkeeping +// action from the maintenance journal. Extracted from storageInsightsService so +// the mapping logic is unit-testable on its own and free of filesystem/db deps. + +import type { DbBreakdownEntry, MaintenanceRunReport } from "../../../shared/types/storage"; + +type DbBreakdownCategoryKey = DbBreakdownEntry["category"]; + +export const DB_BREAKDOWN_META: Record< + DbBreakdownCategoryKey, + { label: string; table: string; action: DbBreakdownEntry["action"] } +> = { + webhooks: { label: "Webhook history", table: "automation_ingress_events", action: "prunable" }, + sync_bookkeeping: { label: "Sync bookkeeping", table: "operations__crsql", action: "compactable" }, + review_artifacts: { label: "Review artifacts", table: "review_run_artifacts", action: "prunable" }, + pr_cache: { label: "PR cache", table: "pull_request_snapshots", action: "prunable" }, + core: { label: "Core data", table: "core", action: null }, +}; + +/** Classify a dbstat table/index name into a coarse storage-breakdown category. */ +export function classifyDbTable(name: string): DbBreakdownCategoryKey { + const lower = name.toLowerCase(); + if (lower.startsWith("operations__crsql")) return "sync_bookkeeping"; + if (lower.includes("automation_ingress_events")) return "webhooks"; + if (lower.includes("review_run_artifacts")) return "review_artifacts"; + if (lower.includes("pull_request_snapshots")) return "pr_cache"; + return "core"; +} + +/** + * Aggregate raw dbstat rows into one breakdown entry per non-empty category. + * `overrides.syncBookkeeping` lets the caller replace the static + * "compactable" label with the journal-derived state (see + * `deriveSyncBookkeepingAction`) so paired projects read "compaction_pending". + */ +export function mapDbBreakdown( + rows: Array<{ name: string; bytes: number }>, + overrides?: { syncBookkeeping?: DbBreakdownEntry["action"] }, +): DbBreakdownEntry[] { + const totals = new Map(); + for (const row of rows) { + if (!row || typeof row.name !== "string") continue; + const bytes = typeof row.bytes === "number" && Number.isFinite(row.bytes) ? row.bytes : 0; + const category = classifyDbTable(row.name); + totals.set(category, (totals.get(category) ?? 0) + Math.max(0, bytes)); + } + const entries: DbBreakdownEntry[] = []; + for (const [category, bytes] of totals) { + if (bytes <= 0) continue; + const meta = DB_BREAKDOWN_META[category]; + const action = category === "sync_bookkeeping" && overrides?.syncBookkeeping !== undefined + ? overrides.syncBookkeeping + : meta.action; + entries.push({ table: meta.table, label: meta.label, bytes, category, action }); + } + return entries.sort((left, right) => right.bytes - left.bytes || left.table.localeCompare(right.table)); +} + +/** + * Sync-bookkeeping compaction state, derived without any new seam. We only + * surface an actionable "Compact now" ("compactable") once the journal proves + * the most recent run's cr-sqlite compaction actually executed without a + * has_peers skip. With no journal yet — or no compact record in the latest run + * — there is no positive evidence that compaction is safe, so we default to + * "compaction_pending" ("Waiting to compact") rather than offer an action that + * might turn out to be peer-blocked. A latest run peer-blocked stays pending. + */ +export function deriveSyncBookkeepingAction( + journal: readonly MaintenanceRunReport[], +): DbBreakdownEntry["action"] { + const lastCompact = journal[0]?.actions.find((action) => action.ledgerId === "db.operations_crsql"); + if (!lastCompact) return "compaction_pending"; + return lastCompact.skippedReason === "has_peers" ? "compaction_pending" : "compactable"; +} diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 10b9ff307..1a847e56f 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -4,12 +4,8 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { MaintenanceRunReport, StorageCleanupPreview, StorageCleanupTarget } from "../../../shared/types/storage"; import { openKvDb, type AdeDb } from "../state/kvDb"; -import { - classifyDbTable, - createStorageInsightsService, - deriveSyncBookkeepingAction, - mapDbBreakdown, -} from "./storageInsightsService"; +import { createStorageInsightsService } from "./storageInsightsService"; +import { classifyDbTable, deriveSyncBookkeepingAction, mapDbBreakdown } from "./storageDbBreakdown"; import { recordLastFailure } from "../runtime/lastFailureStore"; const logger = { @@ -443,10 +439,15 @@ describe("storageInsightsService", () => { writeSized(path.join(stageStale, "artifact.bin"), 50); fs.utimesSync(stageStale, eightDaysAgo, eightDaysAgo); - // Obsolete recovery backup (old, db healthy, no recent open failure). - const backup = path.join(projectRoot, ".ade", "ade.db.recovery-doctor.bak"); - writeSized(backup, 60); - fs.utimesSync(backup, eightDaysAgo, eightDaysAgo); + // Recovery backups: two obsolete copies (old, db healthy, no recent open + // failure). keepLatest: 1 spares the newest; only the older one is reaped. + const backupNewer = path.join(projectRoot, ".ade", "ade.db.recovery-doctor-new.bak"); + const backupOlder = path.join(projectRoot, ".ade", "ade.db.recovery-doctor-old.bak"); + writeSized(backupNewer, 61); + writeSized(backupOlder, 60); + const nineDaysAgo = new Date(Date.now() - 9 * 24 * 60 * 60_000); + fs.utimesSync(backupNewer, eightDaysAgo, eightDaysAgo); + fs.utimesSync(backupOlder, nineDaysAgo, nineDaysAgo); // iOS DerivedData build cache. const derived = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); @@ -464,7 +465,8 @@ describe("storageInsightsService", () => { expect(fs.existsSync(adeTmpStale)).toBe(false); expect(fs.existsSync(adeTmpFresh)).toBe(true); expect(fs.existsSync(stageStale)).toBe(false); - expect(fs.existsSync(backup)).toBe(false); + expect(fs.existsSync(backupOlder)).toBe(false); + expect(fs.existsSync(backupNewer)).toBe(true); expect(fs.existsSync(derived)).toBe(false); // Compression outcome (transparent gzip). expect(fs.existsSync(`${oldTranscript}.gz`)).toBe(true); @@ -619,8 +621,9 @@ describe("storageInsightsService", () => { reclaimedBytes: 0, dbSizeBytes: 1, }); - // No journal yet → compactable. - expect(deriveSyncBookkeepingAction([])).toBe("compactable"); + // No journal yet → nothing proven safe, so we wait rather than offer an + // action that might turn out to be peer-blocked. + expect(deriveSyncBookkeepingAction([])).toBe("compaction_pending"); // Last run was peer-blocked → waiting to compact. expect(deriveSyncBookkeepingAction([ run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: "has_peers" }), diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index ccd93d39f..868960219 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -27,7 +27,6 @@ import { runGit } from "../git/git"; import type { Logger } from "../logging/logger"; import type { AdeDb } from "../state/kvDb"; import { runQuickCheck } from "../state/kvDb"; -import type { KvDbWithMaintenance } from "../state/dbMaintenanceApi"; import { readLastFailure } from "../runtime/lastFailureStore"; import { isEnoentError } from "../shared/utils"; import type { DiskPressureMonitor } from "./diskPressure"; @@ -37,6 +36,8 @@ import { type CompressionRoots, type CompressionSweepSummary, } from "./historyCompression"; +import { deriveSyncBookkeepingAction, mapDbBreakdown } from "./storageDbBreakdown"; +import { appendMaintenanceJournal, readMaintenanceJournal } from "./storageMaintenanceJournal"; import { deriveCategoryPolicyChips } from "./storageLedger"; import { readVolumeSpace } from "./volume"; @@ -51,7 +52,6 @@ const RECOVERY_BACKUP_PATTERN = /(?:\.pre-crsqlite-w1\.bak|\.recovery-.*\.bak)$/ const HISTORY_SWEEP_START_DELAY_MS = 10 * 60_000; const HISTORY_SWEEP_INTERVAL_MS = 24 * 60 * 60_000; const MAINTENANCE_JOURNAL_FILENAME = "storage-doctor-journal.json"; -const MAINTENANCE_JOURNAL_MAX_RUNS = 30; // One completed maintenance run per project per 20 h reaches PostHog; the // storage doctor runs daily, so the dedupe interval collapses repeat runs // (e.g. daily + a manual "Clean up now") into a single analytics event. @@ -115,72 +115,6 @@ export type StorageInsightsServiceOptions = { stagingTmpDir?: string; }; -type DbBreakdownCategoryKey = DbBreakdownEntry["category"]; - -const DB_BREAKDOWN_META: Record< - DbBreakdownCategoryKey, - { label: string; table: string; action: DbBreakdownEntry["action"] } -> = { - webhooks: { label: "Webhook history", table: "automation_ingress_events", action: "prunable" }, - sync_bookkeeping: { label: "Sync bookkeeping", table: "operations__crsql", action: "compactable" }, - review_artifacts: { label: "Review artifacts", table: "review_run_artifacts", action: "prunable" }, - pr_cache: { label: "PR cache", table: "pull_request_snapshots", action: "prunable" }, - core: { label: "Core data", table: "core", action: null }, -}; - -/** Classify a dbstat table/index name into a coarse storage-breakdown category. */ -export function classifyDbTable(name: string): DbBreakdownCategoryKey { - const lower = name.toLowerCase(); - if (lower.startsWith("operations__crsql")) return "sync_bookkeeping"; - if (lower.includes("automation_ingress_events")) return "webhooks"; - if (lower.includes("review_run_artifacts")) return "review_artifacts"; - if (lower.includes("pull_request_snapshots")) return "pr_cache"; - return "core"; -} - -/** - * Aggregate raw dbstat rows into one breakdown entry per non-empty category. - * `overrides.syncBookkeeping` lets the caller replace the static - * "compactable" label with the journal-derived state (see - * `deriveSyncBookkeepingAction`) so paired projects read "compaction_pending". - */ -export function mapDbBreakdown( - rows: Array<{ name: string; bytes: number }>, - overrides?: { syncBookkeeping?: DbBreakdownEntry["action"] }, -): DbBreakdownEntry[] { - const totals = new Map(); - for (const row of rows) { - if (!row || typeof row.name !== "string") continue; - const bytes = typeof row.bytes === "number" && Number.isFinite(row.bytes) ? row.bytes : 0; - const category = classifyDbTable(row.name); - totals.set(category, (totals.get(category) ?? 0) + Math.max(0, bytes)); - } - const entries: DbBreakdownEntry[] = []; - for (const [category, bytes] of totals) { - if (bytes <= 0) continue; - const meta = DB_BREAKDOWN_META[category]; - const action = category === "sync_bookkeeping" && overrides?.syncBookkeeping !== undefined - ? overrides.syncBookkeeping - : meta.action; - entries.push({ table: meta.table, label: meta.label, bytes, category, action }); - } - return entries.sort((left, right) => right.bytes - left.bytes || left.table.localeCompare(right.table)); -} - -/** - * Sync-bookkeeping compaction state, derived without any new seam: if the most - * recent maintenance run skipped its cr-sqlite compaction because the project - * has sync peers, tombstones cannot be reclaimed yet → "compaction_pending" - * (WS-C renders this as "Waiting to compact"). Otherwise — no journal yet, or - * the last compaction actually ran / was not peer-blocked — it is "compactable". - */ -export function deriveSyncBookkeepingAction( - journal: readonly MaintenanceRunReport[], -): DbBreakdownEntry["action"] { - const lastCompact = journal[0]?.actions.find((action) => action.ledgerId === "db.operations_crsql"); - return lastCompact?.skippedReason === "has_peers" ? "compaction_pending" : "compactable"; -} - export function isObsoleteRecoveryBackup( backupPath: string, options: { projectRoot: string; db: AdeDb; now?: number }, @@ -378,46 +312,6 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti return sweepFlight; }; - const isMaintenanceReport = (value: unknown): value is MaintenanceRunReport => { - if (!value || typeof value !== "object") return false; - const report = value as Record; - return typeof report.startedAt === "string" - && typeof report.finishedAt === "string" - && typeof report.trigger === "string" - && Array.isArray(report.actions) - && typeof report.reclaimedBytes === "number"; - }; - - const readMaintenanceJournal = (): MaintenanceRunReport[] => { - let raw: string; - try { - raw = fs.readFileSync(journalPath, "utf8"); - } catch (error) { - if (isEnoentError(error)) return []; - return []; - } - try { - const parsed = JSON.parse(raw); - return Array.isArray(parsed) ? parsed.filter(isMaintenanceReport) : []; - } catch { - return []; - } - }; - - const appendMaintenanceJournal = (report: MaintenanceRunReport): MaintenanceRunReport[] => { - const next = [report, ...readMaintenanceJournal()].slice(0, MAINTENANCE_JOURNAL_MAX_RUNS); - try { - fs.mkdirSync(path.dirname(journalPath), { recursive: true }); - fs.writeFileSync(journalPath, JSON.stringify(next, null, 2)); - } catch (error) { - options.logger.warn("storage.maintenance_journal_write_failed", { - projectRoot, - error: error instanceof Error ? error.message : String(error), - }); - } - return next; - }; - const statSizeOrNull = (targetPath: string): number | null => { try { return fs.statSync(targetPath).size; @@ -575,7 +469,9 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti const tempPath = path.join(stagingTmpRoot, name); if (path.resolve(tempPath) === projectRoot || path.resolve(tempPath) === adeHome) continue; const stat = await lstatOrNull(tempPath); - if (!stat) continue; + // Skip symlinked staging dirs: the reaper's collectors refuse them, so + // showing one here would be an item cleanup could never act on. + if (!stat || stat.isSymbolicLink()) continue; const stale = Date.now() - stat.mtimeMs > STALE_AGE_MS; const entry = await makeItem({ category: "build_release", @@ -708,7 +604,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti if (item.safety === "safe_to_remove") safeReclaimableBytes += item.bytes; } } - const journal = readMaintenanceJournal(); + const journal = readMaintenanceJournal(journalPath); const extras: StorageSnapshotExtras = { dbBreakdown: computeDbBreakdown(deriveSyncBookkeepingAction(journal)), maintenance: { lastRun: journal[0] ?? null, journal }, @@ -954,11 +850,23 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti const collectObsoleteBackups = async (): Promise => { const names = await readdirOrEmpty(layout.adeDir); - const targets: StorageCleanupTarget[] = []; + // Stat every recovery backup so we can always spare the newest one — the + // ledger's keepLatest: 1 guarantee. Only strictly-older backups are reap + // candidates, and only when they classify obsolete. + const backups: Array<{ path: string; mtimeMs: number }> = []; for (const name of names.filter((value) => RECOVERY_BACKUP_PATTERN.test(value))) { const backupPath = path.join(layout.adeDir, name); - if (isObsoleteRecoveryBackup(backupPath, { projectRoot, db: options.db })) { - targets.push({ kind: "recovery_backup", path: backupPath }); + const stat = await lstatOrNull(backupPath); + if (!stat || !stat.isFile()) continue; + backups.push({ path: backupPath, mtimeMs: stat.mtimeMs }); + } + if (backups.length <= 1) return []; + // Newest first; backups[0] is kept no matter what. + backups.sort((left, right) => right.mtimeMs - left.mtimeMs); + const targets: StorageCleanupTarget[] = []; + for (const backup of backups.slice(1)) { + if (isObsoleteRecoveryBackup(backup.path, { projectRoot, db: options.db })) { + targets.push({ kind: "recovery_backup", path: backup.path }); } } return targets; @@ -1024,7 +932,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti // d. DB retention hooks. `maintenance` is attached by WS-A's kvDb; until it // lands the hooks are undefined and each records an "unsupported" skip. - const maintenance = (options.db as unknown as KvDbWithMaintenance).maintenance; + const maintenance = options.db.maintenance; const runDbStep = ( ledgerId: string, kind: MaintenanceActionKind, @@ -1059,7 +967,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti reclaimedBytes, dbSizeBytes: statSizeOrNull(layout.dbPath), }; - appendMaintenanceJournal(report); + appendMaintenanceJournal(journalPath, report, { logger: options.logger, projectRoot }); cachedSnapshot = null; const failedCount = actions.filter((action) => action.error).length; diff --git a/apps/desktop/src/main/services/storage/storageLedger.ts b/apps/desktop/src/main/services/storage/storageLedger.ts index 88ba6dc1e..790839444 100644 --- a/apps/desktop/src/main/services/storage/storageLedger.ts +++ b/apps/desktop/src/main/services/storage/storageLedger.ts @@ -10,6 +10,17 @@ import type { StorageCategoryId, StorageLedgerEntry, } from "../../../shared/types/storage"; +import { + INGRESS_EVENT_MAX_ROWS_PER_PROJECT, + INGRESS_EVENT_RETENTION_MS, + PR_SNAPSHOT_RETENTION_DAYS, + REVIEW_ARTIFACT_RETENTION_DAYS, +} from "../state/dbMaintenanceApi"; + +// The DB retention/count policies below derive from the shared enforcement +// constants so the ledger, the ingress writer, and the kvDb maintenance hooks +// can never drift apart. +const INGRESS_EVENT_RETENTION_DAYS = Math.round(INGRESS_EVENT_RETENTION_MS / (24 * 60 * 60 * 1_000)); export const STORAGE_LEDGER: readonly StorageLedgerEntry[] = [ // --- Project database tables (§1 evidence) ------------------------------- @@ -18,7 +29,7 @@ export const STORAGE_LEDGER: readonly StorageLedgerEntry[] = [ kind: "table", description: "Webhook history — inbound automation events. 99.99% are written and never read.", policyClass: "operational", - policy: { maxAgeDays: 7, maxRows: 2_000 }, + policy: { maxAgeDays: INGRESS_EVENT_RETENTION_DAYS, maxRows: INGRESS_EVENT_MAX_ROWS_PER_PROJECT }, enforcement: "both", }, { @@ -34,7 +45,7 @@ export const STORAGE_LEDGER: readonly StorageLedgerEntry[] = [ kind: "table", description: "Review artifacts — cached code-review results, re-derivable from a fresh run.", policyClass: "derived", - policy: { maxAgeDays: 30 }, + policy: { maxAgeDays: REVIEW_ARTIFACT_RETENTION_DAYS }, enforcement: "both", }, { @@ -42,7 +53,7 @@ export const STORAGE_LEDGER: readonly StorageLedgerEntry[] = [ kind: "table", description: "PR cache — pull-request metadata re-fetchable from GitHub.", policyClass: "derived", - policy: { maxAgeDays: 60 }, + policy: { maxAgeDays: PR_SNAPSHOT_RETENTION_DAYS }, enforcement: "both", }, { diff --git a/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts new file mode 100644 index 000000000..b8687d639 --- /dev/null +++ b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts @@ -0,0 +1,59 @@ +// Read/write helpers for the storage-doctor maintenance journal — a plain, +// rebuildable JSON file (no DB, no CRR) capping the last N runs. Extracted from +// storageInsightsService and parameterized on the journal path + logger so the +// I/O is testable in isolation. + +import fs from "node:fs"; +import path from "node:path"; +import type { Logger } from "../logging/logger"; +import type { MaintenanceRunReport } from "../../../shared/types/storage"; + +export const MAINTENANCE_JOURNAL_MAX_RUNS = 30; + +export function isMaintenanceReport(value: unknown): value is MaintenanceRunReport { + if (!value || typeof value !== "object") return false; + const report = value as Record; + return typeof report.startedAt === "string" + && typeof report.finishedAt === "string" + && typeof report.trigger === "string" + && Array.isArray(report.actions) + && typeof report.reclaimedBytes === "number"; +} + +export function readMaintenanceJournal(journalPath: string): MaintenanceRunReport[] { + let raw: string; + try { + raw = fs.readFileSync(journalPath, "utf8"); + } catch { + // Missing or unreadable journal → treat as empty (rebuildable file). + return []; + } + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed.filter(isMaintenanceReport) : []; + } catch { + return []; + } +} + +export function appendMaintenanceJournal( + journalPath: string, + report: MaintenanceRunReport, + ctx: { logger: Logger; projectRoot: string }, +): MaintenanceRunReport[] { + const next = [report, ...readMaintenanceJournal(journalPath)].slice(0, MAINTENANCE_JOURNAL_MAX_RUNS); + try { + fs.mkdirSync(path.dirname(journalPath), { recursive: true }); + // Write to a sibling temp file then rename over the target so a crash mid- + // write can never leave a truncated/corrupt journal in place. + const tmpPath = `${journalPath}.tmp`; + fs.writeFileSync(tmpPath, JSON.stringify(next, null, 2)); + fs.renameSync(tmpPath, journalPath); + } catch (error) { + ctx.logger.warn("storage.maintenance_journal_write_failed", { + projectRoot: ctx.projectRoot, + error: error instanceof Error ? error.message : String(error), + }); + } + return next; +} diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index c12f63571..a215e3d6c 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -3381,7 +3381,7 @@ contextBridge.exposeInMainWorld("ade", { ipcRenderer.invoke(IPC.storageCompressNow), ), runMaintenanceNow: async (): Promise => - callProjectRuntimeActionOr("storage", "runMaintenanceNow", {}, () => + callProjectRuntimeActionOr("storage", "runMaintenanceNow", { args: {} }, () => ipcRenderer.invoke(IPC.storageRunMaintenanceNow), ), cleanupPreview: async (targets: StorageCleanupTarget[]): Promise => diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.tsx index ae6f07916..6d255ccbb 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.tsx @@ -2,20 +2,13 @@ import React from "react"; import { Archive, ArrowClockwise, - ArrowDown, - ArrowUp, Broom, CaretDown, CaretRight, - ChartLineUp, - Clock, FileZip, FolderDashed, - Gauge, HardDrives, - Pulse, ShieldCheck, - WarningCircle, } from "@phosphor-icons/react"; import { isUrgentDiskPressure, @@ -44,6 +37,9 @@ import { import { SettingsSectionShell } from "./settingsSectionUi"; import { SmartTooltip } from "../ui/SmartTooltip"; import { StorageCleanupDialog, type SafeCleanupPlanConfig } from "./storage/StorageCleanupDialog"; +import { PANEL_STYLE, STORAGE_BRAND } from "./storage/storageUiConstants"; +import { DiagnosticsStrip, TrendArrow } from "./storage/StorageDiagnostics"; +import { MaintenanceJournal } from "./storage/StorageMaintenanceJournal"; import { CATEGORY_META, CATEGORY_ORDER, @@ -54,34 +50,16 @@ import { buildSafeCleanupPlan, categoryPolicyChip, cleanableEntries, - daemonMemoryBytes, dbBreakdownRows, dbSizeSamples, dbSizeTrend, formatApproxBytes, formatBytes, - formatSlowActions, groupLaneItems, - healthChip, - journalEntries, - lastMaintenanceRun, - maintenanceActionLines, - maintenanceHeadline, safeReclaimableBytes, - sparklinePoints, - type DbSizeSample, type Trend, } from "./storage/storageView"; -const STORAGE_BRAND = "#38BDF8"; - -const PANEL_STYLE: React.CSSProperties = { - background: "color-mix(in srgb, var(--color-card) 90%, var(--color-bg) 10%)", - border: "1px solid color-mix(in srgb, var(--color-border) 78%, transparent)", - borderRadius: 12, - padding: 16, -}; - type CompressNow = () => Promise<{ filesCompressed: number; savedBytes: number }>; type RunMaintenanceNow = () => Promise; type GetRuntimeHealth = () => Promise; @@ -91,17 +69,16 @@ function getCompressNow(): CompressNow | undefined { return typeof fn === "function" ? fn : undefined; } -// runMaintenanceNow / getRuntimeHealth are being added on the main side by a -// sibling workstream. Feature-detect them via a narrow cast so this renderer -// compiles and degrades gracefully whether or not the daemon exposes them yet. +// Feature-detect so the renderer degrades gracefully against an older daemon +// that doesn't yet expose these; both are declared on window.ade in global.d.ts. function getRunMaintenanceNow(): RunMaintenanceNow | undefined { - const fn = (window.ade?.storage as { runMaintenanceNow?: unknown } | undefined)?.runMaintenanceNow; - return typeof fn === "function" ? (fn as RunMaintenanceNow) : undefined; + const fn = window.ade?.storage?.runMaintenanceNow; + return typeof fn === "function" ? fn : undefined; } function getRuntimeHealthFn(): GetRuntimeHealth | undefined { - const fn = (window.ade?.app as { getRuntimeHealth?: unknown } | undefined)?.getRuntimeHealth; - return typeof fn === "function" ? (fn as GetRuntimeHealth) : undefined; + const fn = window.ade?.app?.getRuntimeHealth; + return typeof fn === "function" ? fn : undefined; } type CleanupRequest = { title: string; intro: string; targets: StorageCleanupTarget[] }; @@ -149,28 +126,6 @@ function PolicyChip({ label }: { label: string }) { ); } -function TrendArrow({ trend }: { trend: Trend }) { - if (trend === "down") { - return ( - - - - ); - } - if (trend === "up") { - return ( - - - - ); - } - return ( - - – - - ); -} - function BreakdownBar({ categories }: { categories: StorageCategorySnapshot[] }) { const present = categories.filter((category) => category.bytes > 0); const legend = [...present].sort((a, b) => b.bytes - a.bytes); @@ -685,233 +640,6 @@ function DatabaseCard({ ); } -// --------------------------------------------------------------------------- -// Diagnostics strip -// --------------------------------------------------------------------------- - -function DiagnosticTile({ - icon, - label, - value, - sub, - valueColor, -}: { - icon: React.ReactNode; - label: string; - value: React.ReactNode; - sub?: React.ReactNode; - valueColor?: string; -}) { - return ( -
-
- {icon} - {label} -
-
- {value} -
- {sub ?
{sub}
: null} -
- ); -} - -function Sparkline({ samples }: { samples: DbSizeSample[] }) { - const width = 120; - const height = 30; - const points = sparklinePoints(samples, width, height); - if (points.length < 2) return null; - const path = points.map((p) => `${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" "); - return ( - - - - ); -} - -function DiagnosticsStrip({ - extras, - usage, - usageReady, - runtimeHealth, - runtimeHealthAvailable, -}: { - extras: StorageSnapshotExtras | undefined; - usage: AppResourceUsageSnapshot | null; - usageReady: boolean; - runtimeHealth: RuntimeHealthSnapshot | null; - runtimeHealthAvailable: boolean; -}) { - const samples = React.useMemo(() => dbSizeSamples(extras), [extras]); - const latestDb = samples.length > 0 ? samples[samples.length - 1] : null; - const trend = dbSizeTrend(samples); - const daemonMem = daemonMemoryBytes(usage); - const lastRun = lastMaintenanceRun(extras); - const level = usage ? appResourcePressureLevel(usage) : 0; - const health = healthChip(level); - const healthColor = health.tone === "busy" ? COLORS.danger : health.tone === "elevated" ? COLORS.warning : COLORS.success; - - const notAvailable = Not available yet; - - return ( -
-
-
-

- Health & diagnostics -

-
- How the background service is doing on this Mac. -
-
- {usageReady && usage ? ( - - {health.label} - - ) : null} -
- -
- } - label="Database size" - value={ - latestDb ? ( - - {formatBytes(latestDb.bytes)} - {trend ? : null} - - ) : ( - notAvailable - ) - } - sub={samples.length >= 2 ? : latestDb ? "Trend appears after the next cleanup" : undefined} - /> - } - label="Service memory" - value={daemonMem != null ? formatBytes(daemonMem) : notAvailable} - sub={daemonMem != null ? "Resident right now" : undefined} - /> - } - label="Slow responses" - value={ - runtimeHealthAvailable ? ( - 0 ? COLORS.warning : COLORS.textPrimary }}> - {formatSlowActions(runtimeHealth)} - - ) : ( - notAvailable - ) - } - /> - } - label="Last cleanup" - value={lastRun ? {maintenanceHeadline(lastRun)} : notAvailable} - /> -
-
- ); -} - -// --------------------------------------------------------------------------- -// Maintenance journal -// --------------------------------------------------------------------------- - -function MaintenanceJournal({ extras }: { extras: StorageSnapshotExtras | undefined }) { - const runs = React.useMemo(() => journalEntries(extras, 8), [extras]); - const [expanded, setExpanded] = React.useState(false); - if (runs.length === 0) return null; - - return ( -
- - - {expanded ? ( -
- {runs.map((run, index) => ( - - ))} -
- ) : null} -
- ); -} - -function JournalRow({ run }: { run: MaintenanceRunReport }) { - const lines = maintenanceActionLines(run).filter((line) => line.detail !== "nothing to do"); - return ( -
-
- {maintenanceHeadline(run)} -
- {lines.length > 0 ? ( -
- {lines.map((line, index) => ( - - {line.label} · {line.detail} - - ))} -
- ) : null} -
- ); -} - // --------------------------------------------------------------------------- // Section // --------------------------------------------------------------------------- @@ -1154,7 +882,7 @@ export function StorageSection() {
- ADE never automatically deletes your chats, project files, active lanes, or backups. + ADE never automatically deletes your chats, project files, or active lanes. Old backups are removed automatically once a newer one is verified.
) : null} diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx new file mode 100644 index 000000000..9ca37304a --- /dev/null +++ b/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx @@ -0,0 +1,201 @@ +import React from "react"; +import { + ArrowDown, + ArrowUp, + ChartLineUp, + Clock, + Gauge, + Pulse, + WarningCircle, +} from "@phosphor-icons/react"; +import type { + RuntimeHealthSnapshot, + StorageSnapshotExtras, +} from "../../../../shared/types/storage"; +import type { AppResourceUsageSnapshot } from "../../../../shared/types"; +import { appResourcePressureLevel } from "../../../lib/resourcePressure"; +import { COLORS, SANS_FONT, inlineBadge } from "../../lanes/laneDesignTokens"; +import { PANEL_STYLE, STORAGE_BRAND } from "./storageUiConstants"; +import { + daemonMemoryBytes, + dbSizeSamples, + dbSizeTrend, + formatBytes, + formatSlowActions, + healthChip, + lastMaintenanceRun, + maintenanceHeadline, + sparklinePoints, + type DbSizeSample, + type Trend, +} from "./storageView"; + +export function TrendArrow({ trend }: { trend: Trend }) { + if (trend === "down") { + return ( + + + + ); + } + if (trend === "up") { + return ( + + + + ); + } + return ( + + – + + ); +} + +function DiagnosticTile({ + icon, + label, + value, + sub, + valueColor, +}: { + icon: React.ReactNode; + label: string; + value: React.ReactNode; + sub?: React.ReactNode; + valueColor?: string; +}) { + return ( +
+
+ {icon} + {label} +
+
+ {value} +
+ {sub ?
{sub}
: null} +
+ ); +} + +function Sparkline({ samples }: { samples: DbSizeSample[] }) { + const width = 120; + const height = 30; + const points = sparklinePoints(samples, width, height); + if (points.length < 2) return null; + const path = points.map((p) => `${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" "); + return ( + + + + ); +} + +export function DiagnosticsStrip({ + extras, + usage, + usageReady, + runtimeHealth, + runtimeHealthAvailable, +}: { + extras: StorageSnapshotExtras | undefined; + usage: AppResourceUsageSnapshot | null; + usageReady: boolean; + runtimeHealth: RuntimeHealthSnapshot | null; + runtimeHealthAvailable: boolean; +}) { + const samples = React.useMemo(() => dbSizeSamples(extras), [extras]); + const latestDb = samples.length > 0 ? samples[samples.length - 1] : null; + const trend = dbSizeTrend(samples); + const daemonMem = daemonMemoryBytes(usage); + const lastRun = lastMaintenanceRun(extras); + const level = usage ? appResourcePressureLevel(usage) : 0; + const health = healthChip(level); + const healthColor = health.tone === "busy" ? COLORS.danger : health.tone === "elevated" ? COLORS.warning : COLORS.success; + + const notAvailable = Not available yet; + + return ( +
+
+
+

+ Health & diagnostics +

+
+ How the background service is doing on this Mac. +
+
+ {usageReady && usage ? ( + + {health.label} + + ) : null} +
+ +
+ } + label="Database size" + value={ + latestDb ? ( + + {formatBytes(latestDb.bytes)} + {trend ? : null} + + ) : ( + notAvailable + ) + } + sub={samples.length >= 2 ? : latestDb ? "Trend appears after the next cleanup" : undefined} + /> + } + label="Service memory" + value={daemonMem != null ? formatBytes(daemonMem) : notAvailable} + sub={daemonMem != null ? "Resident right now" : undefined} + /> + } + label="Slow responses" + value={ + runtimeHealthAvailable ? ( + 0 ? COLORS.warning : COLORS.textPrimary }}> + {formatSlowActions(runtimeHealth)} + + ) : ( + notAvailable + ) + } + /> + } + label="Last cleanup" + value={lastRun ? {maintenanceHeadline(lastRun)} : notAvailable} + /> +
+
+ ); +} diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageMaintenanceJournal.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageMaintenanceJournal.tsx new file mode 100644 index 000000000..e956ef98c --- /dev/null +++ b/apps/desktop/src/renderer/components/settings/storage/StorageMaintenanceJournal.tsx @@ -0,0 +1,80 @@ +import React from "react"; +import { CaretDown, CaretRight } from "@phosphor-icons/react"; +import type { + MaintenanceRunReport, + StorageSnapshotExtras, +} from "../../../../shared/types/storage"; +import { COLORS, SANS_FONT } from "../../lanes/laneDesignTokens"; +import { PANEL_STYLE } from "./storageUiConstants"; +import { journalEntries, maintenanceActionLines, maintenanceHeadline } from "./storageView"; + +export function MaintenanceJournal({ extras }: { extras: StorageSnapshotExtras | undefined }) { + const runs = React.useMemo(() => journalEntries(extras, 8), [extras]); + const [expanded, setExpanded] = React.useState(false); + if (runs.length === 0) return null; + + return ( +
+ + + {expanded ? ( +
+ {runs.map((run, index) => ( + + ))} +
+ ) : null} +
+ ); +} + +function JournalRow({ run }: { run: MaintenanceRunReport }) { + const lines = maintenanceActionLines(run).filter((line) => line.detail !== "nothing to do"); + return ( +
+
+ {maintenanceHeadline(run)} +
+ {lines.length > 0 ? ( +
+ {lines.map((line, index) => ( + + {line.label} · {line.detail} + + ))} +
+ ) : null} +
+ ); +} diff --git a/apps/desktop/src/renderer/components/settings/storage/storageUiConstants.ts b/apps/desktop/src/renderer/components/settings/storage/storageUiConstants.ts new file mode 100644 index 000000000..3f9b02ab4 --- /dev/null +++ b/apps/desktop/src/renderer/components/settings/storage/storageUiConstants.ts @@ -0,0 +1,14 @@ +import type React from "react"; + +// Shared presentational constants for the Storage settings surface. Extracted +// so the section shell and the split-out Diagnostics/Journal components can +// share them without a circular import back through StorageSection. + +export const STORAGE_BRAND = "#38BDF8"; + +export const PANEL_STYLE: React.CSSProperties = { + background: "color-mix(in srgb, var(--color-card) 90%, var(--color-bg) 10%)", + border: "1px solid color-mix(in srgb, var(--color-border) 78%, transparent)", + borderRadius: 12, + padding: 16, +}; diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts index c26d8b2ac..42562a98d 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts @@ -4,10 +4,12 @@ import type { MaintenanceRunReport, RuntimeHealthSnapshot, StorageCategorySnapshot, + StorageItem, StorageSnapshotExtras, } from "../../../../shared/types/storage"; import type { AppResourceUsageSnapshot } from "../../../../shared/types"; import { + buildCleanupTarget, buildSafeCleanupPlan, categoryPolicyChip, daemonMemoryBytes, @@ -99,12 +101,13 @@ describe("dbBreakdownRows", () => { }); describe("ledgerLabel", () => { - it("maps known ledger ids to plain names", () => { - expect(ledgerLabel("automation.ingress_events")).toBe("Webhook history"); - expect(ledgerLabel("operations.crsql")).toBe("Sync bookkeeping"); - expect(ledgerLabel("review_run_artifacts")).toBe("Review artifacts"); - expect(ledgerLabel("pull_request_snapshots")).toBe("Pull request cache"); - expect(ledgerLabel("transcripts")).toBe("Chat & terminal history"); + it("maps known ledger ids (exact keys) to plain names", () => { + expect(ledgerLabel("db.automation_ingress_events")).toBe("Webhook history"); + expect(ledgerLabel("db.operations_crsql")).toBe("Sync bookkeeping"); + expect(ledgerLabel("db.review_run_artifacts")).toBe("Review artifacts"); + expect(ledgerLabel("db.pull_request_snapshots")).toBe("Pull request cache"); + expect(ledgerLabel("fs.transcripts")).toBe("Chat & terminal history"); + expect(ledgerLabel("fs.ios_derived_data")).toBe("iOS build cache"); }); it("title-cases an unknown id without leaking an identifier", () => { @@ -116,9 +119,9 @@ describe("maintenanceActionLines", () => { it("orders by reclaim and humanizes each action", () => { const report = run({ actions: [ - { ledgerId: "automation.ingress_events", kind: "prune", itemsAffected: 100, bytesReclaimed: 128 * MB, durationMs: 12 }, - { ledgerId: "operations", kind: "compact", itemsAffected: 0, bytesReclaimed: 0, durationMs: 30 }, - { ledgerId: "review_run_artifacts", kind: "delete", itemsAffected: 3, bytesReclaimed: 0, durationMs: 5, error: "nope" }, + { ledgerId: "db.automation_ingress_events", kind: "prune", itemsAffected: 100, bytesReclaimed: 128 * MB, durationMs: 12 }, + { ledgerId: "db.operations_crsql", kind: "compact", itemsAffected: 0, bytesReclaimed: 0, durationMs: 30 }, + { ledgerId: "db.review_run_artifacts", kind: "delete", itemsAffected: 3, bytesReclaimed: 0, durationMs: 5, error: "nope" }, ], }); const lines = maintenanceActionLines(report); @@ -272,5 +275,35 @@ describe("buildSafeCleanupPlan", () => { expect(plan.fsBytes).toBe(300 * MB); expect(plan.fsGroup?.rows).toHaveLength(1); expect(plan.whatHappens.at(-1)).toContain("never touched"); + expect(plan.whatHappens.at(-1)).toContain("newest backup is always kept"); + }); +}); + +describe("buildCleanupTarget", () => { + const item = (p: string): StorageItem => ({ + id: p, + label: "x", + path: p, + bytes: 1, + fileCount: 1, + lastModifiedAt: null, + safety: "safe_to_remove", + }); + + it("maps a direct .ade/tmp child to stale_tmp_staging", () => { + expect(buildCleanupTarget("build_release", item("/proj/.ade/tmp/ios-testflight-1"), new Map())) + .toEqual({ kind: "stale_tmp_staging", path: "/proj/.ade/tmp/ios-testflight-1" }); + }); + + it("does not map a deeper (grandchild) .ade/tmp path", () => { + expect(buildCleanupTarget("build_release", item("/proj/.ade/tmp/ios-testflight-1/nested"), new Map())) + .toBeNull(); + }); + + it("still maps system ade-* staging and .ade/cache paths", () => { + expect(buildCleanupTarget("build_release", item("/tmp/ade-run-9"), new Map())) + .toEqual({ kind: "stale_tmp_staging", path: "/tmp/ade-run-9" }); + expect(buildCleanupTarget("caches", item("/proj/.ade/cache/npm"), new Map())) + .toEqual({ kind: "rebuildable_cache", path: "/proj/.ade/cache/npm" }); }); }); diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.ts index 56278b50b..54255f9a7 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.ts @@ -103,6 +103,11 @@ export function baseName(p: string): string { const CACHE_SEGMENT = /[\\/]\.ade[\\/]cache[\\/]/; const TMP_STAGING = /[\\/]ade-[^\\/]*[\\/]?$/; +// A direct child of the project-relative `.ade/tmp` release-staging dir (e.g. +// `.ade/tmp/ios-testflight-123`). The backend validates these under the same +// `stale_tmp_staging` kind as the system-temp `ade-*` dirs; a deeper path (a +// grandchild) intentionally does not match. +const ADE_TMP_STAGING = /[\\/]\.ade[\\/]tmp[\\/][^\\/]+$/; /** * Resolve the cleanup target for a single item, or null when it cannot be safely @@ -134,6 +139,7 @@ export function buildCleanupTarget( case "build_release": case "caches": if (CACHE_SEGMENT.test(p)) return { kind: "rebuildable_cache", path: p }; + if (ADE_TMP_STAGING.test(p)) return { kind: "stale_tmp_staging", path: p }; if (TMP_STAGING.test(p)) return { kind: "stale_tmp_staging", path: p }; return null; default: @@ -274,31 +280,37 @@ export function dbBreakdownRows(entries: DbBreakdownEntry[] | undefined): DbBrea /** Copy shown in the "waiting to compact" tooltip. Never references sync internals. */ export const DB_COMPACTION_PENDING_HINT = - "ADE will reclaim this automatically once none of your devices are mid-sync."; + "Kept while devices stay in sync. Safe to leave — a future update will reclaim it."; // ---- Maintenance journal --------------------------------------------------- /** - * Best-effort friendly name for a ledger id (e.g. "automation.ingress_events" → - * "Webhook history"). Falls back to a title-cased version of the last segment so - * an unknown ledger entry still reads cleanly and never leaks an identifier. + * Friendly name for a ledger id, keyed by the exact ids declared in + * `storageLedger.ts` (and used verbatim as maintenance-action `ledgerId`s). + * Unknown ids fall back to a title-cased last segment so an entry we haven't + * labelled still reads cleanly and never leaks a raw identifier. */ -const LEDGER_LABEL_RULES: Array<{ match: RegExp; label: string }> = [ - { match: /ingress|webhook/i, label: "Webhook history" }, - { match: /operation|tombstone|bookkeep|\bcrr\b|crsql/i, label: "Sync bookkeeping" }, - { match: /review/i, label: "Review artifacts" }, - { match: /pull_?request|pr[_.]?(cache|snapshot)|snapshot/i, label: "Pull request cache" }, - { match: /transcript|chat|terminal|history/i, label: "Chat & terminal history" }, - { match: /tmp|staging|temp/i, label: "Temporary files" }, - { match: /backup|recovery/i, label: "Recovery backups" }, - { match: /cache/i, label: "Caches" }, - { match: /vacuum|freelist|database|\.db|\bdb\b/i, label: "Database file" }, -]; +const LEDGER_LABELS: Record = { + "db.automation_ingress_events": "Webhook history", + "db.operations_crsql": "Sync bookkeeping", + "db.review_run_artifacts": "Review artifacts", + "db.pull_request_snapshots": "Pull request cache", + "db.core": "Core data", + "fs.transcripts": "Chat & terminal history", + "fs.tmp": "Release staging", + "fs.tmp_staging": "Build staging", + "fs.recovery_backups": "Recovery backups", + "fs.cache": "Caches", + "fs.ios_derived_data": "iOS build cache", + "fs.storage_doctor_journal": "Maintenance journal", + "fs.artifacts": "Proof & recordings", + "fs.attachments": "Attachments", + "fs.worktrees": "Lane worktrees", +}; export function ledgerLabel(ledgerId: string): string { - for (const rule of LEDGER_LABEL_RULES) { - if (rule.match.test(ledgerId)) return rule.label; - } + const known = LEDGER_LABELS[ledgerId]; + if (known) return known; const segment = ledgerId.split(/[.:/]/).pop() ?? ledgerId; const words = segment.replace(/[_-]+/g, " ").trim(); if (!words) return "Maintenance"; @@ -558,14 +570,12 @@ export function buildSafeCleanupPlan( const fsGroup: SafeCleanupGroup | null = fsRows.length > 0 ? { heading: "Temporary & rebuildable files", rows: fsRows } : null; - if (fsGroup) { - groups.push(fsGroup); - whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); - } else if (estimatedBytes > 0) { + if (fsGroup) groups.push(fsGroup); + if (fsGroup || estimatedBytes > 0) { whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); } - whatHappens.push("Your chats, projects, active lanes, and backups are never touched."); + whatHappens.push("Your chats, projects, and active lanes are never touched, and your newest backup is always kept."); return { fsTargets, fsBytes, groups, fsGroup, whatHappens, estimatedBytes }; } From 8048a277633f9ac7015c106ddf72c12bd13df006 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:02:51 -0400 Subject: [PATCH 03/10] test(storage): consolidate ledger suite; docs + CLI parity for the storage overhaul - fold storageLedger.test.ts into storageInsightsService.test.ts (folder test budget), suite green - docs: storage-and-recovery/automations/settings READMEs + ARCHITECTURE rewritten to current reality (doctor sweep, ledger, maintenance hooks, WAL, diagnostics strip, new IPC channels) - ade-cli: typed `ade storage maintenance` subcommand + formatter + README inventory (generic actions path already resolved runMaintenanceNow) - iOS + TUI parity verified not-applicable (no surface / already reachable) Co-Authored-By: Claude Fable 5 --- apps/ade-cli/README.md | 1 + apps/ade-cli/src/cli.test.ts | 40 ++++++ apps/ade-cli/src/cli.ts | 52 ++++++- .../storage/storageInsightsService.test.ts | 72 ++++++++++ .../services/storage/storageLedger.test.ts | 73 ---------- docs/ARCHITECTURE.md | 40 ++++-- docs/features/automations/README.md | 4 +- .../automations/triggers-and-actions.md | 3 +- .../onboarding-and-settings/README.md | 23 +-- docs/features/storage-and-recovery/README.md | 135 +++++++++++++++++- 10 files changed, 338 insertions(+), 105 deletions(-) delete mode 100644 apps/desktop/src/main/services/storage/storageLedger.test.ts diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index 36e487017..155d37686 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -438,6 +438,7 @@ ade usage budget cumulative --scope global --text ade storage snapshot --text # categorized ADE disk usage + free space (mirrors the desktop storage dashboard) ade storage snapshot --refresh --text # force a fresh scan instead of the cached snapshot ade storage compress --text # losslessly compress old chat/terminal history +ade --role cto storage maintenance --text # run the policy-driven ledger maintenance sweep now (CTO) ade storage actions --text # raw storage service actions (cleanupPreview/cleanup live here) ade actions list --domain chat --text ade actions run git.stageFile --arg laneId=lane-id --arg path=src/index.ts diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index 37cf43a83..5f5406047 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -1590,6 +1590,15 @@ describe("ADE CLI", () => { arguments: { domain: "storage", action: "compressNow", args: {} }, }); + const maintenancePlan = expectExecutePlan( + buildCliPlan(["storage", "maintenance"]), + ); + expect(inferFormatter(maintenancePlan)).toBe("storage-maintenance"); + expect(maintenancePlan.steps[0]?.params).toEqual({ + name: "run_ade_action", + arguments: { domain: "storage", action: "runMaintenanceNow", args: {} }, + }); + expect( expectExecutePlan(buildCliPlan(["storage", "actions"])).steps[0]?.params, ).toEqual({ name: "list_ade_actions", arguments: { domain: "storage" } }); @@ -1639,6 +1648,37 @@ describe("ADE CLI", () => { expect(snapshotText).toContain("ADE storage"); expect(snapshotText).toContain("GB free of"); expect(snapshotText).toContain("chats_history"); + + const maintenanceText = formatOutput( + { + startedAt: "2026-07-12T00:00:00.000Z", + finishedAt: "2026-07-12T00:00:01.000Z", + trigger: "manual", + reclaimedBytes: 3 * 1024 ** 2, + dbSizeBytes: 45 * 1024 ** 2, + actions: [ + { + ledgerId: "automation_ingress_events", + kind: "prune", + itemsAffected: 120, + bytesReclaimed: 2 * 1024 ** 2, + }, + { + ledgerId: "pull_request_snapshots", + kind: "vacuum", + itemsAffected: 0, + bytesReclaimed: 0, + skippedReason: "not due", + }, + ], + }, + { text: true } as never, + inferFormatter(maintenancePlan), + ); + expect(maintenanceText).toContain("ADE storage maintenance"); + expect(maintenanceText).toContain("manual"); + expect(maintenanceText).toContain("automation_ingress_events"); + expect(maintenanceText).toContain("skipped: not due"); }); it("formats external session action results as text", () => { diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 123d3fadb..90b570d6e 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -233,6 +233,7 @@ type FormatterId = | "external-sessions" | "storage-snapshot" | "storage-compress" + | "storage-maintenance" | "sync-status" | "sync-web"; @@ -2087,12 +2088,14 @@ const HELP_BY_COMMAND: Record = { Reports what ADE is holding on disk (chats/terminal history, lane worktrees, build output, caches, proof attachments, recovery backups, database) and the volume's free space, mirroring the desktop Settings storage dashboard. The - snapshot is read-only; compression is lossless and safe. Target-scoped cleanup + snapshot is read-only; compression is lossless and safe. Maintenance runs the + policy-driven ledger sweep (prune/compress/vacuum). Target-scoped cleanup (which deletes files) is intentionally left to the action bridge. $ ade storage snapshot --text Categorized ADE disk usage + free-space summary $ ade storage snapshot --refresh --text Force a fresh scan (skip the cached snapshot) $ ade storage compress --text Losslessly compress old chat/terminal history + $ ade --role cto storage maintenance --text Run the policy-driven maintenance sweep now (CTO) $ ade storage actions --text List raw storage service actions $ ade storage action cleanupPreview --input-json '{"targets":[...]}' Preview a target-scoped cleanup $ ade --role cto storage action cleanup --input-json '{"targets":[...],"preview":{...}}' Delete previewed targets (CTO) @@ -10221,8 +10224,19 @@ function buildStoragePlan(args: string[]): CliPlan { steps: [actionStep("result", "storage", "compressNow", {})], }; } + if (sub === "maintenance" || sub === "maintain" || sub === "run-maintenance") { + // Runs the same policy-driven maintenance sweep as the desktop Settings + // "Run maintenance now" button (prune/compress/vacuum per the storage + // ledger). CTO-only at the action bridge, so agents must pass --role cto. + return { + kind: "execute", + label: "storage maintenance", + formatter: "storage-maintenance", + steps: [actionStep("result", "storage", "runMaintenanceNow", {})], + }; + } throw new CliUsageError( - "storage supports snapshot, compress, actions, or action . Use 'ade actions run storage.cleanupPreview' / 'storage.cleanup' for target-scoped cleanup.", + "storage supports snapshot, compress, maintenance, actions, or action . Use 'ade actions run storage.cleanupPreview' / 'storage.cleanup' for target-scoped cleanup.", ); } @@ -16327,6 +16341,38 @@ function formatStorageCompression(value: unknown): string { ]); } +function formatStorageMaintenance(value: unknown): string { + if (!isRecord(value)) return JSON.stringify(value, null, 2); + const header = renderKeyValues("ADE storage maintenance", [ + ["trigger", value.trigger], + ["reclaimed", formatBytes(value.reclaimedBytes)], + ["db size", typeof value.dbSizeBytes === "number" ? formatBytes(value.dbSizeBytes) : undefined], + ["started", value.startedAt], + ["finished", value.finishedAt], + ]); + const actions = Array.isArray(value.actions) ? value.actions.filter(isRecord) : []; + const rows = actions.map((action) => { + const note = typeof action.error === "string" && action.error + ? `error: ${action.error}` + : typeof action.skippedReason === "string" && action.skippedReason + ? `skipped: ${action.skippedReason}` + : ""; + return [ + cell(action.ledgerId, 32), + cell(action.kind, 12), + typeof action.itemsAffected === "number" ? String(action.itemsAffected) : "-", + formatBytes(action.bytesReclaimed), + note, + ]; + }); + const table = renderTable( + ["LEDGER", "KIND", "ITEMS", "RECLAIMED", "NOTE"], + rows, + "No maintenance actions were applied.", + ); + return `${header}\n\n${table}`; +} + function formatLastFailureLine(report: AdeLastFailureReport): string { const repeat = report.count > 1 ? ` x${report.count}` : ""; const scope = report.projectRoot ? ` [${report.projectRoot}]` : ""; @@ -17884,6 +17930,8 @@ function formatTextOutput( return formatStorageSnapshot(value); case "storage-compress": return formatStorageCompression(value); + case "storage-maintenance": + return formatStorageMaintenance(value); case "action-result": default: if (isRecord(value)) diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 1a847e56f..ef84bf889 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -6,6 +6,13 @@ import type { MaintenanceRunReport, StorageCleanupPreview, StorageCleanupTarget import { openKvDb, type AdeDb } from "../state/kvDb"; import { createStorageInsightsService } from "./storageInsightsService"; import { classifyDbTable, deriveSyncBookkeepingAction, mapDbBreakdown } from "./storageDbBreakdown"; +import { ADE_LAYOUT_DEFINITIONS } from "../../../shared/adeLayout"; +import { + deriveCategoryPolicyChips, + getLedgerEntry, + LEDGER_LAYOUT_COVERAGE, + STORAGE_LEDGER, +} from "./storageLedger"; import { recordLastFailure } from "../runtime/lastFailureStore"; const logger = { @@ -643,3 +650,68 @@ describe("storageInsightsService", () => { ])).toBe("compactable"); }); }); + +describe("storageLedger", () => { + it("declares a well-formed, unique policy for every entry", () => { + const ids = new Set(); + for (const entry of STORAGE_LEDGER) { + expect(entry.id).toMatch(/^[a-z]+\.[a-z0-9_]+$/); + expect(ids.has(entry.id)).toBe(false); + ids.add(entry.id); + expect(entry.description.length).toBeGreaterThan(0); + expect(["table", "directory", "file_family"]).toContain(entry.kind); + expect(["user_data", "derived", "operational"]).toContain(entry.policyClass); + expect(["write_time", "doctor", "both", "manual"]).toContain(entry.enforcement); + // compressAfterDays only applies to user_data (spec §2 contract). + if (entry.policy.compressAfterDays != null) { + expect(entry.policyClass).toBe("user_data"); + } + } + }); + + it("covers every §1 evidence table and the doctor's own artifacts", () => { + for (const id of [ + "db.automation_ingress_events", + "db.operations_crsql", + "db.review_run_artifacts", + "db.pull_request_snapshots", + "fs.transcripts", + "fs.tmp", + "fs.recovery_backups", + "fs.cache", + "fs.storage_doctor_journal", + "fs.artifacts", + "fs.attachments", + ]) { + expect(getLedgerEntry(id), `missing ledger entry ${id}`).toBeDefined(); + } + }); + + it("declares a storage policy (or explicit unmanaged waiver) for every layout directory", () => { + // CI guard: adding a new tracked directory to ADE_LAYOUT_DEFINITIONS without + // declaring its storage policy here must fail. Either map it to a ledger id + // or add it to LEDGER_LAYOUT_COVERAGE as intentionally unmanaged (null). + const ledgerIds = new Set(STORAGE_LEDGER.map((entry) => entry.id)); + for (const definition of ADE_LAYOUT_DEFINITIONS) { + if (definition.pathType !== "directory") continue; + const declared = Object.prototype.hasOwnProperty.call( + LEDGER_LAYOUT_COVERAGE, + definition.relativePath, + ); + expect(declared, `layout dir ${definition.relativePath} has no ledger coverage entry`).toBe(true); + const ledgerId = LEDGER_LAYOUT_COVERAGE[definition.relativePath]; + if (ledgerId !== null && ledgerId !== undefined) { + expect(ledgerIds.has(ledgerId), `coverage points at missing ledger id ${ledgerId}`).toBe(true); + } + } + }); + + it("derives category policy chips from the ledger policy values", () => { + const chips = deriveCategoryPolicyChips(); + expect(chips.chats_history).toBe("Compressed after 14 days"); + expect(chips.build_release).toBe("Auto-cleans after 7 days"); + expect(chips.recovery_backups).toBe("Keeps the latest backup"); + expect(chips.caches).toBeTruthy(); + expect(chips.lanes_worktrees).toBeTruthy(); + }); +}); diff --git a/apps/desktop/src/main/services/storage/storageLedger.test.ts b/apps/desktop/src/main/services/storage/storageLedger.test.ts deleted file mode 100644 index 8b1bf47b3..000000000 --- a/apps/desktop/src/main/services/storage/storageLedger.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { ADE_LAYOUT_DEFINITIONS } from "../../../shared/adeLayout"; -import { - deriveCategoryPolicyChips, - getLedgerEntry, - LEDGER_LAYOUT_COVERAGE, - STORAGE_LEDGER, -} from "./storageLedger"; - -describe("storageLedger", () => { - it("declares a well-formed, unique policy for every entry", () => { - const ids = new Set(); - for (const entry of STORAGE_LEDGER) { - expect(entry.id).toMatch(/^[a-z]+\.[a-z0-9_]+$/); - expect(ids.has(entry.id)).toBe(false); - ids.add(entry.id); - expect(entry.description.length).toBeGreaterThan(0); - expect(["table", "directory", "file_family"]).toContain(entry.kind); - expect(["user_data", "derived", "operational"]).toContain(entry.policyClass); - expect(["write_time", "doctor", "both", "manual"]).toContain(entry.enforcement); - // compressAfterDays only applies to user_data (spec §2 contract). - if (entry.policy.compressAfterDays != null) { - expect(entry.policyClass).toBe("user_data"); - } - } - }); - - it("covers every §1 evidence table and the doctor's own artifacts", () => { - for (const id of [ - "db.automation_ingress_events", - "db.operations_crsql", - "db.review_run_artifacts", - "db.pull_request_snapshots", - "fs.transcripts", - "fs.tmp", - "fs.recovery_backups", - "fs.cache", - "fs.storage_doctor_journal", - "fs.artifacts", - "fs.attachments", - ]) { - expect(getLedgerEntry(id), `missing ledger entry ${id}`).toBeDefined(); - } - }); - - it("declares a storage policy (or explicit unmanaged waiver) for every layout directory", () => { - // CI guard: adding a new tracked directory to ADE_LAYOUT_DEFINITIONS without - // declaring its storage policy here must fail. Either map it to a ledger id - // or add it to LEDGER_LAYOUT_COVERAGE as intentionally unmanaged (null). - const ledgerIds = new Set(STORAGE_LEDGER.map((entry) => entry.id)); - for (const definition of ADE_LAYOUT_DEFINITIONS) { - if (definition.pathType !== "directory") continue; - const declared = Object.prototype.hasOwnProperty.call( - LEDGER_LAYOUT_COVERAGE, - definition.relativePath, - ); - expect(declared, `layout dir ${definition.relativePath} has no ledger coverage entry`).toBe(true); - const ledgerId = LEDGER_LAYOUT_COVERAGE[definition.relativePath]; - if (ledgerId !== null && ledgerId !== undefined) { - expect(ledgerIds.has(ledgerId), `coverage points at missing ledger id ${ledgerId}`).toBe(true); - } - } - }); - - it("derives category policy chips from the ledger policy values", () => { - const chips = deriveCategoryPolicyChips(); - expect(chips.chats_history).toBe("Compressed after 14 days"); - expect(chips.build_release).toBe("Auto-cleans after 7 days"); - expect(chips.recovery_backups).toBe("Keeps the latest backup"); - expect(chips.caches).toBeTruthy(); - expect(chips.lanes_worktrees).toBeTruthy(); - }); -}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1bbb804b6..2048e753e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -189,7 +189,7 @@ The desktop app is a **client of the runtime**. It owns a trusted main process, **Runtime binding pools.** -- `apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts` — desktop-side client for the local brain endpoint. Spawns or attaches to the machine endpoint, registers local projects with `projects.add`, dispatches local runtime actions, applies short per-call timeouts for project registration / file actions / event polling, emits a `local_runtime.action_slow` warning log when an action call exceeds 500 ms or throws (the log breaks the total into `ensureProjectMs` / `connectMs` / `daemonCallMs` so a stalled renderer call is debuggable before the IPC timeout fires), and best-effort installs the background service in packaged builds. Local project windows use this binding consistently outside unit tests. +- `apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts` — desktop-side client for the local brain endpoint. Spawns or attaches to the machine endpoint, registers local projects with `projects.add`, dispatches local runtime actions, applies short per-call timeouts for project registration / file actions / event polling, emits a `local_runtime.action_slow` warning log when an action call exceeds 500 ms or throws (the log breaks the total into `ensureProjectMs` / `connectMs` / `daemonCallMs` so a stalled renderer call is debuggable before the IPC timeout fires) and records that call into a bounded (age + 5,000-sample) rolling 24 h window surfaced as `getRuntimeHealth()` (count + nearest-rank p95) behind the `ade.app.getRuntimeHealth` IPC, and best-effort installs the background service in packaged builds. Local project windows use this binding consistently outside unit tests. - `apps/desktop/src/main/services/remoteRuntime/` — paired/SSH runtime pool. `remoteTargetRegistry.ts` stores saved machines under `~/.ade/secrets/remote-machines.json` (manual host plus an optional `routes[]` of Tailscale / Bonjour / manual addresses with per-route `lastSucceededAt` and manual-disconnect state); `sshTransport.ts` handles ssh-agent / key based transport with bounded connect/exec timeouts, normalized handshake errors, disabled SSH keepalives, and alternate routes ranked by most-recent success; `remoteBootstrap.ts` does first-connect runtime upload + version/hash negotiation against the bundled `ade-` binary, native deps, and PTY host worker, and walks alternate ADE channel homes (`.ade` / `.ade-alpha` / `.ade-beta`) when the preferred runtime is missing **or fails validated initialization** (including a just-uploaded Beta), retaining the winning home for follow-up commands such as the JSON-stdin SSH-to-paired upgrade; version / channel / capability skew becomes `compatibilityWarnings`; `remoteConnectionPool.ts` keeps the per-window remote runtime binding alive, gates `projects.*` runtime calls against the connection's `capabilities.machineProjects` flags (missing capabilities reject the call with a self-describing error), reconnects safely on read-only actions (`get*/list*/read*/search*/diagnosticsGet*` and a small allowlist) after genuine connection failures, owns local TCP forwards for remote preview ports, memoizes optional-action fallbacks, and emits eviction notifications when the paired/SSH transport or JSON-RPC client closes; `remoteConnectionService.ts` listens for those evictions, marks targets as `error`, preserves explicit manual disconnects, pauses automatic reconnect after repeated implicit failures, surfaces the capabilities + compatibility warnings on every `RemoteRuntimeConnectionStatus`, and re-probes saved connections on `powerMonitor.resume` / `unlock-screen`; `runtimeRpcClient.ts` is the JSON-RPC client (per-call timeouts expire only the matching request without replay, while transport close/error or malformed JSON framing rejects every pending call; bounded oversized responses reject only the matching request, and remote errors preserve the original method plus the JSON-RPC `code` / `message` / `data`); `runtimeDiscovery.ts` runs Bonjour + Tailscale in parallel and returns `{ machines, diagnostics }` so a missing or stuck `tailscale` CLI does not silently swallow LAN discovery. Build outputs (configured in `apps/desktop/tsup.config.ts`): @@ -273,7 +273,7 @@ ADE uses Node's native `node:sqlite` driver (no better-sqlite3 dependency) with - **Engine source**: `apps/desktop/src/main/services/state/kvDb.ts` (schema bootstrap, CRR enablement, sync API) and `crsqliteExtension.ts` (extension loader). Both the desktop main process and the ADE CLI runtime import the same engine module from here; they do not maintain parallel schemas. The database is owned by whichever process opened it first for a given project; in normal desktop operation that owner is the ADE runtime, while desktop in-process users are limited to pre-binding flows, diagnostics, tests, and Electron-only work. - **Database file**: `/.ade/ade.db`. -- **WAL mode** handles durability; `flushNow()` is a no-op. +- **WAL mode**: `openRawDatabase` sets `PRAGMA journal_mode = WAL` + `PRAGMA synchronous = NORMAL` at open. `flushNow()` forces pending WAL frames onto the main file with a `wal_checkpoint(TRUNCATE)` (used before shutdown and after a vacuum). - **CRRs**: eligible tables are marked via `SELECT crsql_as_crr('table_name')` at startup. Virtual/internal tables (`sqlite_%`, `crsql_%`) are excluded. Marking is dynamic — new tables are picked up automatically unless excluded. - **Sync API** (`AdeDb.sync`): `getSiteId()`, `getDbVersion()`, `exportChangesSince(version)`, `applyChanges(changes)`. Used by the sync transport. - **Merge semantics**: last-writer-wins per column with Lamport timestamps; each device has a site ID at `.ade/secrets/sync-site-id`. @@ -284,6 +284,14 @@ ADE uses Node's native `node:sqlite` driver (no better-sqlite3 dependency) with open failures flow through `lastFailureStore` and the brain-independent `projectRecoveryService`. See [Storage and recovery](./features/storage-and-recovery/README.md). +- **Maintenance hooks**: `openKvDb` attaches an optional `maintenance` + (`DbMaintenanceApi`, from `state/dbMaintenanceApi.ts`) handle — retention + prunes for `automation_ingress_events` / `review_run_artifacts` / + `pull_request_snapshots`, a zero-peers-only cr-sqlite tombstone compaction, and + a fragmentation-gated vacuum that activates `auto_vacuum = INCREMENTAL` so later + sweeps stay bounded. The storage doctor in `storageInsightsService` invokes + these on a schedule; the retention constants are the single source of truth + imported by the ingress writer and the storage ledger alike. ### 3.2 Schema highlights @@ -359,10 +367,15 @@ Types for these tables are split into domain modules under `apps/desktop/src/sha The storage domain lives under `apps/desktop/src/main/services/storage/`: `diskPressure` gates new write-producing work, `storageInsightsService` provides categorized and -preview-confirmed cleanup without following links, and `historyCompression` -compresses inactive history only after byte-for-byte verification. Renderer -surfaces are `StoragePressureIndicator`, `StorageSection`, and -`ProjectRecoveryScreen`. +preview-confirmed cleanup without following links plus the scheduled **storage +doctor** maintenance sweep (`runMaintenanceNow` + post-boot/daily timers), and +`historyCompression` compresses inactive history only after byte-for-byte +verification. `storageLedger` declares the bounding policy for every table and +directory (with a CI coverage cross-check against `ADE_LAYOUT_DEFINITIONS`) and +the doctor journals each run to `.ade/cache/storage-doctor-journal.json`. +Renderer surfaces are `StoragePressureIndicator`, `StorageSection` (with its +diagnostics strip + maintenance journal), and `ProjectRecoveryScreen`. See +[Storage and recovery](./features/storage-and-recovery/README.md). **Project scaffold modes.** `initializeOrRepairAdeProject(projectRoot, { mode })` controls whether a project gets the full shared scaffold or stays local-only: @@ -479,15 +492,16 @@ Related feature docs: [Chat](./features/chat/README.md), [Agents](./features/age `apps/desktop/src/shared/ipc.ts` defines the single `IPC` const with ~550 named channel strings in a `ade..` namespace: ``` -ade.app.* # app lifecycle, clipboard text and image (writeClipboardText, writeClipboardImage, saveClipboardImageAttachment), paths, image data-URL preview (getImageDataUrl), the deeplink navigation push channel ade.app.navigate (AppNavigationRequest payloads from the ade:// protocol handler, the ade code app/navigate JSON-RPC, and the iOS deeplinks.open sync command — see features/deeplinks/README.md), the one-way zoom push channel ade.app.zoomCommand (AppZoomCommand "in"/"out"/"reset" sent from the native View menu to the renderer's window.ade.zoom.onCommand so menu/keyboard zoom shares the in-app zoom path — display %, persistence, and the macOS traffic-light inset), and the resource-pressure snapshot ade.app.getResourceUsage (async, coalesced `AppResourceUsageSnapshot` backing the TopBar pressure indicator: one bounded/timeout-guarded `ps` sample shared across windows behind a 900 ms cache + in-flight coalescing, with disjoint per-role attribution built in `services/pty/resourceUsageSampling.ts` — see features/terminals-and-sessions/pty-and-processes.md) +ade.app.* # app lifecycle, clipboard text and image (writeClipboardText, writeClipboardImage, saveClipboardImageAttachment), paths, image data-URL preview (getImageDataUrl), the deeplink navigation push channel ade.app.navigate (AppNavigationRequest payloads from the ade:// protocol handler, the ade code app/navigate JSON-RPC, and the iOS deeplinks.open sync command — see features/deeplinks/README.md), the one-way zoom push channel ade.app.zoomCommand (AppZoomCommand "in"/"out"/"reset" sent from the native View menu to the renderer's window.ade.zoom.onCommand so menu/keyboard zoom shares the in-app zoom path — display %, persistence, and the macOS traffic-light inset), and the resource-pressure snapshot ade.app.getResourceUsage (async, coalesced `AppResourceUsageSnapshot` backing the TopBar pressure indicator: one bounded/timeout-guarded `ps` sample shared across windows behind a 900 ms cache + in-flight coalescing, with disjoint per-role attribution built in `services/pty/resourceUsageSampling.ts` — see features/terminals-and-sessions/pty-and-processes.md), and the machine-level daemon-health snapshot ade.app.getRuntimeHealth (async `RuntimeHealthSnapshot` — a rolling 24 h count + p95 of slow/errored local-runtime action calls, read directly off `localRuntimeConnectionPool` with no action-domain routing, feeding the Storage > Diagnostics "slow responses" tile) ade.project.* # project open/close/switch/state, unified local+remote recents (listRecent, key-based forget/reorder, setRecentPinned), in-app directory browser (browseDirectories, getDetail), git path inspection (inspectPath — ProjectPathInspection behind the renderer's worktree-open gate; promise-cached in services/projects/projectPathInspector.ts with a `fresh` bypass and invalidated on lane attach/adopt from both the in-process handler and the runtime-bridge action path), favicon resolver/override (resolveIcon, chooseIcon, removeIcon) with local-only filesystem allowlists. openRepo/switchToPath surface AdeRecoveryErrorCode-coded failures (via surfaceCodedError) so the renderer can route a failed open into the recovery screen ade.recovery.* # brain-independent project-open recovery: diagnose / repair # (projectRecoveryService against projectRecoveryConnectionPool). # See features/storage-and-recovery/README.md ade.storage.* # disk-pressure snapshot (getPressure) + storage dashboard: - # getSnapshot / compressNow / cleanupPreview / cleanup, backed by - # diskPressureMonitor + storageInsightsService and the `storage` - # ADE action domain (cleanup is CTO-only) + # getSnapshot / compressNow / runMaintenanceNow / cleanupPreview / + # cleanup, backed by diskPressureMonitor + storageInsightsService + # (runMaintenanceNow triggers the storage-doctor sweep) and the + # `storage` ADE action domain (cleanup + runMaintenanceNow are CTO-only) ade.onboarding.* ade.lanes.* # lane list/create/delete/stack/template/env/port/proxy/rebase # delete pipeline: ade.lanes.delete + ade.lanes.delete.cancel @@ -643,7 +657,7 @@ Most services described here live under `apps/desktop/src/main/services/ | `keybindings/` | `keybindingsService.ts` | User keybindings read/write. | | `lanes/` | `laneService.ts`, `laneEnvironmentService.ts`, `laneTemplateService.ts`, `laneProxyService.ts`, `portAllocationService.ts`, `autoRebaseService.ts`, `rebaseSuggestionService.ts`, `laneLaunchContext.ts`, `oauthRedirectService.ts`, `runtimeDiagnosticsService.ts` | Worktree lifecycle, env bootstrap, templates, reverse proxy, port leases, auto-rebase, suggestions, OAuth redirect, diagnostics. | | `logging/` | `logger.ts` | File-backed structured logger. | -| `localRuntime/` | `localRuntimeConnectionPool.ts` | Desktop-side client for the local brain endpoint. Spawns or attaches to the machine endpoint, registers local projects with `projects.add`, dispatches local runtime actions with per-call timeouts where needed, emits `local_runtime.action_slow` warn logs (with `ensureProjectMs` / `connectMs` / `daemonCallMs` breakdown) whenever a call exceeds 500 ms or throws, polls/subscribes to runtime events while preserving `eventEpoch`/gap metadata, and installs the background service best-effort in packaged builds. | +| `localRuntime/` | `localRuntimeConnectionPool.ts` | Desktop-side client for the local brain endpoint. Spawns or attaches to the machine endpoint, registers local projects with `projects.add`, dispatches local runtime actions with per-call timeouts where needed, emits `local_runtime.action_slow` warn logs (with `ensureProjectMs` / `connectMs` / `daemonCallMs` breakdown) whenever a call exceeds 500 ms or throws and aggregates those calls into a bounded rolling 24 h window exposed as `getRuntimeHealth()` (count + p95) for the `ade.app.getRuntimeHealth` IPC / Storage diagnostics tile, polls/subscribes to runtime events while preserving `eventEpoch`/gap metadata, and installs the background service best-effort in packaged builds. | | `onboarding/` | `onboardingService.ts`, `onboardingSuggestedConfig.ts` | First-run flow, defaults detection, existing lane discovery. `onboardingSuggestedConfig.ts` contains pure workflow parsing and suggested `.ade/ade.yaml` generation. | | `opencode/` | `openCodeRuntime.ts`, `openCodeServerManager.ts`, `openCodeBinaryManager.ts`, `openCodeInventory.ts`, `openCodeModelCatalog.ts` | OpenCode server spawn, binary resolution, model discovery. | | `orchestration/` | `orchestrationService.ts`, `applyPatches.ts`, `patchPolicy.ts`, `manifestNormalization.ts`, `runtimeProfile.ts` | Work-tab orchestration for multi-phase plans. `orchestrationService` manages run lifecycle, manifest persistence, the `leadState.planning` state machine, `plan.md`, validation strategy/findings, asset bundles, and the `lineage` delegation ledger (lead→worker/validator spawn + result edges). `patchPolicy` keeps privileged fields (`leadState.planning`, `planSpec`, and the `/lineage` ledger) behind service methods so the lead cannot forge intake, planning rounds, model routing, approval readiness, or delegation edges with a raw patch. `runtimeProfile` resolves the active orchestration profile per session and gates model selection / plan approval on planning readiness. The renderer surfaces live in `renderer/components/orchestration/` (see §7.3). The former `orchestrator/` and `missions/` directories were consolidated into this service. | @@ -656,11 +670,11 @@ Most services described here live under `apps/desktop/src/main/services/ | `search/` | `searchService.ts`, `searchIndexDb.ts`, `searchQueryParser.ts`, `searchRanking.ts`, `terminalChunking.ts`, `searchServiceWiring.ts` | Universal search over chat/terminal/PR/commit/branch text via a disposable FTS5 index (`.ade/cache/search-index.db`, never inside `ade.db`, never synced), unioned at query time with delegated lanes/files/artifacts/Linear. Debounced off-hot-path ingestion with cursor-based incremental reads, deterministic ranking tiers, and the `search` ADE action domain (`query`/`indexStatus`/`rebuildIndex`). `searchServiceWiring.ts` is shared with the `ade` runtime so wiring can't drift. See [features/search/README.md](./features/search/README.md). | | `sessions/` | `sessionService.ts`, `sessionDeltaService.ts` | Terminal session CRUD, post-session delta computation. | | `shared/` | `utils.ts`, `imageDimensions.ts`, `queueRebase.ts`, `packLegacyUtils.ts`, `transcriptInsights.ts` | Cross-domain utilities, including shared record guards and PNG/JPEG dimension parsing used by App Control and iOS Simulator capture paths. | -| `state/` | `kvDb.ts`, `crsqliteExtension.ts`, `globalState.ts`, `projectState.ts`, `onConflictAudit.ts` | SQLite schema + open, CRR extension loader, global state file, per-project state init. `globalState.upsertRecentProject` accepts `preserveRecentOrder` so reactivating an already-known project (by app focus, deep link, etc.) refreshes its `lastOpenedAt` in place instead of jumping it to the front of the recents list. Recent projects use stable keys: local rows are keyed by absolute root path, remote rows by `remote::`, so a remote path string never collides with a local project. Pinned rows are retained above normal recency ordering and survive beyond the cap. `model_picker_favorites` and `model_picker_recents` are per-project CRR tables shared by desktop, TUI, and iOS; they are primary-key-only so CRR can convert them, with the recents cap enforced in `modelPickerStore.ts`. `AdeDb.sync.discardUnpublishedChangesForTables(tableNames)` lets a service clear local CRR state for specific tables without leaking those clears to sync peers — it records the cleared tables and `through_db_version` in the local-only `local_crr_change_suppressions` table, and `exportChangesSince` filters local-site rows for those tables at or below that version on the way out. The local-only excluded set (still kept out of replication) includes that suppression table itself, the snapshot caches, `local_worktree_residual_cleanups`, `pr_auto_link_ignores`, `pull_request_ai_summaries`, and `runtime_processes`. `crsql_changes` DELETE statements run through a helper that swallows the read-only-table error the cr-sqlite extension raises when a CRR-managed table is wiped, with a `db.crr_changes_cleanup_skipped` warn log instead of failing the migration. | +| `state/` | `kvDb.ts`, `crsqliteExtension.ts`, `dbMaintenanceApi.ts`, `globalState.ts`, `projectState.ts`, `onConflictAudit.ts` | SQLite schema + open (WAL + `synchronous = NORMAL`), CRR extension loader, global state file, per-project state init. `kvDb` also attaches the optional `maintenance` (`DbMaintenanceApi`) handle — retention prunes, zero-peers-only cr-sqlite compaction, and fragmentation-gated vacuum — whose interface and shared retention constants live in `dbMaintenanceApi.ts` and are invoked by the storage doctor. `globalState.upsertRecentProject` accepts `preserveRecentOrder` so reactivating an already-known project (by app focus, deep link, etc.) refreshes its `lastOpenedAt` in place instead of jumping it to the front of the recents list. Recent projects use stable keys: local rows are keyed by absolute root path, remote rows by `remote::`, so a remote path string never collides with a local project. Pinned rows are retained above normal recency ordering and survive beyond the cap. `model_picker_favorites` and `model_picker_recents` are per-project CRR tables shared by desktop, TUI, and iOS; they are primary-key-only so CRR can convert them, with the recents cap enforced in `modelPickerStore.ts`. `AdeDb.sync.discardUnpublishedChangesForTables(tableNames)` lets a service clear local CRR state for specific tables without leaking those clears to sync peers — it records the cleared tables and `through_db_version` in the local-only `local_crr_change_suppressions` table, and `exportChangesSince` filters local-site rows for those tables at or below that version on the way out. The local-only excluded set (still kept out of replication) includes that suppression table itself, the snapshot caches, `local_worktree_residual_cleanups`, `pr_auto_link_ignores`, `pull_request_ai_summaries`, and `runtime_processes`. `crsql_changes` DELETE statements run through a helper that swallows the read-only-table error the cr-sqlite extension raises when a CRR-managed table is wiped, with a `db.crr_changes_cleanup_skipped` warn log instead of failing the migration. | | `sync/` | `syncService.ts`, `syncHostService.ts`, `syncPeerService.ts`, `syncRemoteCommandService.ts`, `syncProtocol.ts`, `deviceRegistryService.ts`, `syncPairingStore.ts` | **Thin delegation to the ADE runtime's sync service.** The authoritative sync service now lives in `apps/ade-cli/src/services/sync/`; the desktop main-process instances default to a non-host viewer role for legacy state and tests. The old in-process host is disabled unless `ADE_ENABLE_DESKTOP_SYNC_HOST=1` (diagnostics only). Wire formats — WebSocket envelope, remote command routing, device registry, pairing secrets — are the same across both implementations. Viewer joins clear the local `devices` + `sync_cluster_state` rows and then call `db.sync.discardUnpublishedChangesForTables(["devices", "sync_cluster_state"])` so the resulting DELETE rows do not leak back to other peers; the peer client follows up with `syncPeerService.acknowledgeLocalDbVersion()` to advance the outbound cursor past the suppressed range. | | `tests/` | `testService.ts` | Test-suite execution + run history. | | `updates/` | `autoUpdateService.ts` | Electron auto-update wrapper around `electron-updater`. Owns the renderer-visible `AutoUpdateSnapshot` (`idle \| checking \| downloading \| ready \| installing \| error`), uses `compareUpdateVersions` (SemVer-aware) to dedupe / supersede staged installers and to reconcile `pendingInstallUpdate` against the running version on next boot. Packaged builds schedule startup/periodic checks; source/dev launches construct the service without auto-check timers so missing `app-update.yml` never surfaces as a renderer error. ADE manually starts downloads after a cache-volume capacity preflight, checks the installed-app volume again before staging, classifies disk/quota/network/verification/permission/installer failures in the shared snapshot, preserves verified downloads when safe, and bounds the native installer handoff with a watchdog. `quitAndInstall()` re-checks the staged version before calling `updater.quitAndInstall(false, true)`. See [desktop auto-update disk-space behavior](./features/onboarding-and-settings/desktop-auto-update.md). | -| `storage/` | `diskPressure.ts`, `volume.ts`, `storageInsightsService.ts`, `historyCompression.ts` | Disk-full/recovery hardening. `diskPressure` samples all ADE storage roots, classifies pressure with recovery hysteresis, and gates write-producing operation classes via `canPerform(kind)` (enforced at each start boundary in `agentChatService` / `ptyService` / `processService` and the compressor). `storageInsightsService` builds the categorized Settings > Storage snapshot and preview-confirmed, link-safe cleanup. `historyCompression` losslessly gzip-compresses inactive old transcripts/logs after byte-identity verification and exposes the transparent `.gz` read/reinflate helpers used by transcript, session, and search readers. Constructed in both `main.ts` and the `ade` runtime `bootstrap.ts`. See [features/storage-and-recovery/README.md](./features/storage-and-recovery/README.md). | +| `storage/` | `diskPressure.ts`, `volume.ts`, `storageInsightsService.ts`, `historyCompression.ts`, `storageLedger.ts`, `storageDbBreakdown.ts`, `storageMaintenanceJournal.ts` | Disk-full/recovery hardening + the storage doctor. `diskPressure` samples all ADE storage roots, classifies pressure with recovery hysteresis, and gates write-producing operation classes via `canPerform(kind)` (enforced at each start boundary in `agentChatService` / `ptyService` / `processService` and the compressor). `storageInsightsService` builds the categorized Settings > Storage snapshot and preview-confirmed, link-safe cleanup, and runs the scheduled **storage doctor** maintenance sweep (`runMaintenanceNow` + post-boot/daily timers) that compresses history, reaps safe staging/backups/iOS build data, and invokes the kvDb DB-maintenance hooks. `storageLedger` is the declared bounding policy for every table/directory (with a CI coverage cross-check against `ADE_LAYOUT_DEFINITIONS`); `storageDbBreakdown` maps `dbstat` rows into the project-database breakdown; `storageMaintenanceJournal` reads/writes the 30-run doctor journal. `historyCompression` losslessly gzip-compresses inactive old transcripts/logs after byte-identity verification and exposes the transparent `.gz` read/reinflate helpers. Constructed in both `main.ts` and the `ade` runtime `bootstrap.ts`. See [features/storage-and-recovery/README.md](./features/storage-and-recovery/README.md). | | `usage/` | `usageTrackingService.ts`, `providerQuotaParsers.ts`, `usageStatsStore.ts`, `budgetCapService.ts`, `ledgers/localUsageLedgers.ts` | Live provider quota/cost accounting, budget enforcement, and retrospective activity stats. `usageTrackingService.ts` owns polling, pacing, provider/GitHub cache orchestration, and `getAdeUsageStats`; `providerQuotaParsers.ts` normalizes Claude and Codex quota payload variants and classifies Codex windows by advertised duration rather than assuming the provider's primary/secondary positions. The stats read returns cached expensive sources plus live project-DB aggregates immediately, marks the result `refreshing` when stale, and revalidates provider ledgers / GitHub in the background. `usageStatsStore.ts` aggregates AI calls, sessions, lanes, code movement, artifacts, automations, workers, streaks, and the local-only cross-client `usage_events` ledger. Local provider scanners live under `usage/ledgers/`. Budget caps can match a rule scope while `usd-per-run` evaluates usage records keyed to the active run id. For runtime-backed projects, the machine brain is the sole quota poller and the renderer consumes its pushed snapshot; `main.ts` does not start a competing project-context tracker. Threshold state remains shared at module level for the unbound/local contexts, and `main.ts` adds a final IPC-level dedup gate with a 10-minute TTL per `provider:threshold:resetCycle` key. | | `perf/` | `perfLog.ts`, `perfIpc.ts`, `metricsSampler.ts`, `aggregator.ts` | Opt-in local performance harness. `ADE_PERF_RUN_ID` opens a JSONL event log, samples Electron process metrics, records IPC durations, accepts renderer perf marks/web-vitals, and aggregates each run into `summary.json`. | diff --git a/docs/features/automations/README.md b/docs/features/automations/README.md index f0f2912ed..3a024ab00 100644 --- a/docs/features/automations/README.md +++ b/docs/features/automations/README.md @@ -22,7 +22,7 @@ Caveat: GitHub-polling and webhook ingress only work on a runtime that can reach These services are loaded by the ADE runtime's project scope (and by the desktop main process when it hosts a local project) — the path reflects the source tree, not where the code "runs". -- `automationService.ts` — main service. Rule CRUD, execution dispatch (`agent-session`, `built-in`), cron scheduling (via `node-cron`), durable deferred-lane cleanup, lane lifecycle dispatch, file-change watching (via `chokidar`), queue management, run history, confidence scoring, billing codes, ingress cursor storage. +- `automationService.ts` — main service. Rule CRUD, execution dispatch (`agent-session`, `built-in`), cron scheduling (via `node-cron`), durable deferred-lane cleanup, lane lifecycle dispatch, file-change watching (via `chokidar`), queue management, run history, confidence scoring, billing codes, ingress cursor storage. **Ingress retention:** it no longer persists raw webhook payloads (`raw_payload_json` is always written `null`), and on every insert it prunes `automation_ingress_events` for that project at write time — dropping rows older than 7 days and, for non-`dispatched` rows, everything beyond the newest 2,000 (insert + prune run in one `BEGIN IMMEDIATE`). At construction it runs a one-time, chunked reclaim that nulls any legacy `raw_payload_json` still on disk and prunes the review-artifact / PR-snapshot tables to their retention windows; a reclaim failure is logged and retried next boot, never blocking service startup. The retention/count bounds are imported from `state/dbMaintenanceApi` so the writer, the storage-doctor DB hooks, and the storage ledger enforce one policy. - `automationPlannerService.ts` — natural-language rule authoring. `parseNaturalLanguage`, `validateDraft`, `saveDraft`, `simulate`. Runs a planner subprocess (Claude or Codex) to turn a free-text brief into an `AutomationRuleDraft`. - `automationIngressService.ts` — HTTP webhook ingress (GitHub, custom webhooks) plus GitHub relay cursor drains and a repo-scoped WebSocket wake-up subscription. Signature verification for webhooks. `AutomationIngressEventRecord` is the normalized event shape. Accepts `automationService: null` for the **PR-freshness-only mode** described under [Runtime ownership](#runtime-ownership): the relay still feeds `prService.ingestGithubWebhook`, but rule dispatch, the local webhook server, and ingress status/event reads are skipped. In that mode the relay cursor is persisted through an injected `ingressCursorStore` — `createKvIngressCursorStore(db)`, which reads/writes `automations.ingress.cursor.` in the kv table — instead of `automationService`'s cursor storage. A missing GitHub App user or ADE account token puts both polling and socket connection attempts into a quiet 5-minute auth-pending cooldown (relay status `disabled`, a single `automations.github_relay_auth_pending` info log) rather than warning every tick; `pollNow()` bypasses the cooldown. `stop()`/`dispose()` abort active polling and close the socket plus polling/connect/reconnect timers. - `githubPollingService.ts` — direct GitHub REST polling for the origin repo plus `extraRepos`. Each tick first asks `automationService.hasEnabledGithubRules()` (or the injected equivalent) and does no GitHub work unless at least one enabled rule has a canonical `github.*` trigger. Active ticks diff per-poll snapshots of issues/PRs/comments to emit `github.issue_*` and `github.pr_*` events without requiring a webhook or relay. Cursor format is `=|=` to support multi-repo state in a single stored string; see `readCursor`/`writeCursor` for the legacy-compat parser. @@ -152,7 +152,7 @@ Automations accept inbound events from four sources (`AutomationIngressSource`): - `linear-relay` — Linear event relay for automation triggers; Linear triggers here are context-only. Two delivery modes share the relay: a per-workspace webhook created by `linearIngressService.setup()` (requires a workspace-admin credential; per-org signing secret registered in the Worker's D1), or the ADE Linear OAuth app (`linearAppClient.ts` — client id bundled, PKCE, `read,write,admin` scope), whose webhook Linear auto-provisions on authorization and signs with the app-level `LINEAR_APP_WEBHOOK_SECRET` the Worker holds. App-connected projects self-configure on the first poll (`isAdeAppConnection` dep) — no manual connect step; teardown never deletes the app's webhook. - `github-polling` — `githubPollingService` polls the GitHub REST API directly for the origin repo and any `extraRepos`, diffing per-poll snapshots to synthesize `github.issue_*` / `github.pr_*` events (opened / edited / labeled / closed / commented, and PR merged). No relay or webhook infra required. Cursor is a `=|=` string stored via `automationService.setIngressCursor({ source: "github-polling" })`; default interval is 30s, but each tick returns before network access when no enabled rule has a `github.*` trigger. -Ingress events normalize to `AutomationIngressEventRecord` with `source`, `eventKey`, `triggerType`, `summary`, plus `cursor` for relay/polling replay. Matching rules are resolved by `eventKey`-to-rule-id mapping. An optional `repo` filter on a rule's trigger (e.g. `github.issue_opened` with `repo: "owner/name"`) restricts dispatch when multiple repos are polled. +Ingress events normalize to `AutomationIngressEventRecord` with `source`, `eventKey`, `triggerType`, `summary`, plus `cursor` for relay/polling replay. Matching rules are resolved by `eventKey`-to-rule-id mapping. An optional `repo` filter on a rule's trigger (e.g. `github.issue_opened` with `repo: "owner/name"`) restricts dispatch when multiple repos are polled. The record no longer carries the raw payload — ADE keeps only the normalized fields — and the `automation_ingress_events` row is bounded at write time (7 days, and the newest 2,000 non-`dispatched` rows per project), with the storage doctor sweeping the same table on its schedule. See [Storage and recovery](../storage-and-recovery/README.md#database-maintenance-hooks). Enabling an externally triggered rule is capability-gated by the paths that can actually fire it, computed once per status read by `computeDeliveryStatuses`. It resolves the live availability of each path (relay configured, GitHub polling capable, local webhook listening/healthy, public gateway ready, Linear ingress capable) and returns an `AutomationIngressDelivery` — one `AutomationTriggerDeliveryStatus` (`ready`, the winning `via` path, and a human `setupError`) per source class: `github`, `githubWebhook`, `webhook`, and `linear`. Canonical `github.*` rules are ready if any of relay, direct polling, local webhook, or public gateway is available (in that preference order); `github-webhook` needs relay, local webhook, or public gateway; custom `webhook` needs local webhook or public gateway; `linear.*` needs the injected Linear-ingress capability. `triggerDeliveryKeyForType(type)` (`shared/types/automations.ts`) maps a trigger type to its delivery key (`github.*` and legacy `git.pr_*` → `github`, `linear.*` → `linear`, `github-webhook` → `githubWebhook`, `webhook` → `webhook`, everything else → `null`), and `getIngressSetupError(rule)` returns the first not-ready source's `setupError` — so the message points to the corresponding Automations setup rather than requiring ADE Webhook Gateway for every external trigger. `getIngressStatus()` returns the whole `AutomationIngressDelivery` as an optional `delivery` field (optional for compatibility with remote runtimes built before per-trigger delivery reporting); the renderer reads it to gate the `TriggerCard`/`RuleRow` delivery callouts. There is no separate "does this rule need external ingress" allowlist to keep in sync — the reconcile loop calls `getIngressSetupError` for every enabled rule and lets a `null` result mean "no external ingress required". diff --git a/docs/features/automations/triggers-and-actions.md b/docs/features/automations/triggers-and-actions.md index e59082830..24d4612ce 100644 --- a/docs/features/automations/triggers-and-actions.md +++ b/docs/features/automations/triggers-and-actions.md @@ -115,10 +115,11 @@ Lane lifecycle `namePattern` values are globs matched against the resolved lane - `triggerType` — maps to one of the trigger types above. - `status` — `AutomationIngressStatus` (`received`, `matched`, `dispatched`, `ignored`, `error`). - `summary` — human-readable one-liner. -- `rawPayloadJson` — the full original payload. - `cursor` — for relay polling. - `receivedAt`. +The raw webhook payload is **not** persisted or returned. `automationIngressService` normalizes and matches against the payload in memory, but the stored `automation_ingress_events` row keeps only the fields above (`raw_payload_json` is written `null`), and the record `automationService` returns to run-detail / history readers no longer includes a `rawPayloadJson`. Rows are bounded at write time (7 days, and the newest 2,000 non-`dispatched` rows per project) and swept again by the storage doctor. + Label normalization helper `normalizeLabels` accepts either string arrays or objects with a `.name` property (the GitHub payload shape). ## Tool palettes diff --git a/docs/features/onboarding-and-settings/README.md b/docs/features/onboarding-and-settings/README.md index 0d982d350..9c06884c4 100644 --- a/docs/features/onboarding-and-settings/README.md +++ b/docs/features/onboarding-and-settings/README.md @@ -294,15 +294,22 @@ Renderer — settings: and `LaneBehaviorSection.tsx` — lane initialization recipes and lifecycle policies. - `apps/desktop/src/renderer/components/settings/StorageSection.tsx` plus - `settings/storage/StorageCleanupDialog.tsx` and `settings/storage/storageView.ts` + `settings/storage/StorageCleanupDialog.tsx`, `StorageDiagnostics.tsx`, + `StorageMaintenanceJournal.tsx`, `storageView.ts`, and `storageUiConstants.ts` — Settings > Storage. Renders the current volume-pressure state, a category - breakdown of ADE's on-disk footprint, and per-category removable items; - offers preview-confirmed cleanup and a manual history-compression action. - `storageView.ts` holds the pure snapshot→cleanup-target mapping and category - presentation metadata (unit-tested without a DOM); the dashboard reads - `window.ade.storage.*`. The domain lives in + breakdown of ADE's on-disk footprint with per-category policy chips, and + per-category removable items; a "Clean up safely" primary action that runs the + storage doctor (`runMaintenanceNow`, falling back to preview-confirmed + cleanup-by-target on older daemons); a project-database breakdown card with + per-row prune/compact actions; the "Health & diagnostics" strip + (`StorageDiagnostics`, anchored `#diagnostics`); and the collapsible recent-cleanups + journal (`StorageMaintenanceJournal`). `storageView.ts` holds the pure + snapshot→cleanup-target mapping plus the diagnostics/maintenance view-model + (unit-tested without a DOM); the dashboard reads `window.ade.storage.*` and + `window.ade.app.getRuntimeHealth`. The domain lives in [Storage and recovery](../storage-and-recovery/README.md), which owns the - disk-pressure monitor and `storageInsightsService` behind those IPCs. + disk-pressure monitor, `storageInsightsService`, and the storage doctor behind + those IPCs. - `apps/desktop/src/renderer/components/settings/SyncDevicesSection.tsx` — shared multi-device sync management used by the focused **Connections > Phone** and **Connections > Web** tabs beneath a shared @@ -614,7 +621,7 @@ changing rather than which service backs it: | AI Connections | `ProvidersSection.tsx` | Provider CLIs, models, API-key status, provider readiness, OpenCode runtime diagnostics. When Claude is installed but unauthenticated, the shared `Login to Claude` CTA opens a primary-lane terminal running `claude auth login` and navigates to Work. Legacy `?tab=providers` lands here. | | Background Jobs | `AiFeaturesSection.tsx` | AI-powered automations: summaries, PR descriptions, commit messages, auto-naming, plus project-wide scheduled-work recovery. **Pause all scheduled work** keeps Claude wakeups, cron tasks, and loops armed while suppressing `nextWakeAt`; on resume each overdue schedule runs once before cron work returns to its normal cadence. **Active scheduled work** lists KV-backed durable jobs from every chat with per-job Cancel and an explicit unavailable/error state. Legacy `?tab=automations` lands here. Each feature row has an independent reasoning-effort override (`ReasoningEffortPicker` with `useFamilyDefaults={false}`). | | Lane Templates | `LaneTemplatesSection.tsx`, `LaneBehaviorSection.tsx` | Lane init recipes and lane lifecycle policy | -| Storage | `StorageSection.tsx`, `storage/StorageCleanupDialog.tsx`, `storage/storageView.ts` | Disk-usage dashboard: current volume pressure, ADE storage broken down by category (lanes/worktrees, chats & terminal history, caches, build & release, proof & attachments, recovery backups, database), preview-confirmed cleanup of the removable subset, and a manual "compress old history" action. Reads `window.ade.storage.getPressure` / `getSnapshot` and mutates through `compressNow` / `cleanupPreview` / `cleanup`. Deep links from `?tab=storage` and `?tab=disk` (via `TAB_ALIASES`). See [Storage and recovery](../storage-and-recovery/README.md). | +| Storage | `StorageSection.tsx`, `storage/StorageCleanupDialog.tsx`, `storage/StorageDiagnostics.tsx`, `storage/StorageMaintenanceJournal.tsx`, `storage/storageView.ts` | Disk-usage dashboard: current volume pressure, ADE storage broken down by category (lanes/worktrees, chats & terminal history, caches, build & release, proof & attachments, recovery backups, database) with policy chips, a project-database breakdown, a "Clean up safely" action that runs the storage doctor, a Health & diagnostics strip, and the recent-cleanups journal — plus preview-confirmed cleanup and a manual "compress old history" action. Reads `window.ade.storage.getPressure` / `getSnapshot` and `window.ade.app.getRuntimeHealth`, and mutates through `runMaintenanceNow` / `compressNow` / `cleanupPreview` / `cleanup`. Deep links from `?tab=storage` and `?tab=disk` (via `TAB_ALIASES`); the top-bar load pill deep-links to `?tab=storage#diagnostics` (`?tab=diagnostics` also aliases here). See [Storage and recovery](../storage-and-recovery/README.md). | | Stats | `AdeUsageSection.tsx`, `ActivityModule.tsx`, `providerColors.ts` | Usage page with live Limits plus a sectioned Activity dashboard: overview stat tiles, an activity/tokens/code/clients module, and split AI-usage and GitHub-vs-local Code & PRs panels, with project/machine scope and day/week/month/year/all ranges. Fast cached local-provider, project-DB, GitHub, and cross-client activity. Deep links from `?tab=usage` and `?tab=stats` land here. | > Live provider quota windows and automation guardrails live in the top-bar Usage popup (`HeaderUsageControl.tsx` → `UsageQuotaPanel.tsx` + collapsible `BudgetCapEditor`) and Settings > Usage > Limits. The Activity tab is the retrospective cross-client dashboard. diff --git a/docs/features/storage-and-recovery/README.md b/docs/features/storage-and-recovery/README.md index c69a2df15..59ae74dd8 100644 --- a/docs/features/storage-and-recovery/README.md +++ b/docs/features/storage-and-recovery/README.md @@ -4,7 +4,8 @@ | Path | Role | |---|---| -| `apps/desktop/src/main/services/state/kvDb.ts` | Opens the project database, runs the interrupted-rebuild recovery pass, classifies database-open errors, creates the headroom-gated migration backup, and exports `rebuildTableInTransaction` / `recoverInterruptedTableRebuilds`. | +| `apps/desktop/src/main/services/state/kvDb.ts` | Opens the project database (enabling `journal_mode = WAL` + `synchronous = NORMAL` at open), runs the interrupted-rebuild recovery pass, classifies database-open errors, creates the headroom-gated migration backup, and exports `rebuildTableInTransaction` / `recoverInterruptedTableRebuilds`. Attaches the optional `maintenance` (`DbMaintenanceApi`) handle — the prune / compact / vacuum hooks the storage doctor invokes. | +| `apps/desktop/src/main/services/state/dbMaintenanceApi.ts` | The `DbMaintenanceApi` interface consumed by the storage doctor, plus the single source of truth for the DB retention/count bounds (`INGRESS_EVENT_RETENTION_MS` = 7 days, `INGRESS_EVENT_MAX_ROWS_PER_PROJECT` = 2,000, `REVIEW_ARTIFACT_RETENTION_DAYS` = 30, `PR_SNAPSHOT_RETENTION_DAYS` = 60) imported by the ingress writer, the kvDb hooks, and the storage ledger so the policy can never drift across enforcement sites. | | `apps/desktop/src/main/services/state/durableFile.ts` | Atomic temp-write-and-rename persistence, one-generation `.lkg` JSON backup, validation, and primary/previous recovery reads. | | `apps/desktop/src/main/services/chat/agentChatService.ts` | Persists chat metadata and transcripts, records provider-pointer transitions to the bounded thread-pointer ledger, reconciles missing pointers from ledger/resume command/transcript, gates new turns on disk pressure (`canPerform("chat_turn")`), and implements explicit `recoverContinuity` modes. | | `apps/desktop/src/main/services/chat/threadPointerLedger.ts` | Standalone append-only continuity ledger (`thread-pointers.jsonl`): typed `ThreadPointerLedgerEntry` records, tolerant parse that drops only a torn tail line, newest-per-session read, and 64 KiB self-compaction (newest records first) via an atomic rewrite. | @@ -15,16 +16,22 @@ | `apps/desktop/src/main/services/runtime/projectRecoveryService.ts` | Brain-independent diagnosis and ordered repair: space, ownership, database validation, migration recovery, service restart, endpoint/project verification, and chat reconciliation. | | `apps/desktop/src/main/services/storage/diskPressure.ts` | Samples all ADE storage roots, classifies pressure with recovery hysteresis, and gates write-producing operation classes via `canPerform(kind)`. Exports the `DiskPressureMonitor` type and refusal-message copy. | | `apps/desktop/src/main/services/storage/volume.ts` | `readVolumeSpace(dir)` (statfs free/total bytes) and `isNoSpaceError(err)` (ENOSPC/EDQUOT and disk-full message detection), shared by the pressure monitor and the database-open error classifier. | -| `apps/desktop/src/main/services/storage/storageInsightsService.ts` | Builds categorized storage snapshots and preview-confirmed cleanup plans without following symlinks or deleting protected state. | +| `apps/desktop/src/main/services/storage/storageInsightsService.ts` | Builds categorized storage snapshots and preview-confirmed cleanup plans without following symlinks or deleting protected state. Also runs the **storage doctor**: `runMaintenanceNow()` plus the post-boot (10 min) + daily (24 h) timers that compress history, auto-reap safe staging / obsolete backups / iOS DerivedData, and invoke the kvDb DB-maintenance hooks, writing each run to the maintenance journal and emitting one deduped `ade_feature_used` analytics event. Populates the snapshot's optional `extras` (db breakdown, journal, policy chips, safe-reclaimable total). | +| `apps/desktop/src/main/services/storage/storageLedger.ts` | The **storage ledger** (`STORAGE_LEDGER`): the declared policy for every persistent table and directory ADE writes — its privacy class (`user_data` / `derived` / `operational`) and how it is bounded (`write_time` / `doctor` / `both` / `manual`). `LEDGER_LAYOUT_COVERAGE` maps every `ADE_LAYOUT_DEFINITIONS` directory to a ledger id (or `null` for intentionally-unmanaged config/credentials) so a coverage test fails CI if a new tracked directory ships without a declared policy. `deriveCategoryPolicyChips()` renders the Settings policy chips from the ledger. | +| `apps/desktop/src/main/services/storage/storageDbBreakdown.ts` | Pure helpers turning raw `dbstat` rows into the coarse project-database breakdown (`classifyDbTable` / `mapDbBreakdown`: webhooks, sync bookkeeping, review artifacts, PR cache, core) and `deriveSyncBookkeepingAction` — which reads the journal so the sync-bookkeeping row offers "Compact now" only after a run proves compaction ran without a `has_peers` skip, and stays "waiting to compact" otherwise. | +| `apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts` | Read/write helpers for the storage-doctor journal — a plain rebuildable JSON file (`storage-doctor-journal.json` under `.ade/cache`, no DB/CRR) capping the last 30 runs, written via temp-file-then-rename so a crash never leaves a torn journal. | | `apps/desktop/src/main/services/storage/historyCompression.ts` | Finds inactive old history, gzip-compresses it, verifies byte identity, and only then removes the original; also reinflates before append. Exposes `readHistoryFileSync` / `reinflateHistoryFileSync` so transcript, session, and search readers can read a `.gz` generation transparently. | | `apps/desktop/src/renderer/components/app/StoragePressureIndicator.tsx` | Quiet top-right warning/critical/exhausted status and entry point to Storage settings. Mounted in `TopBar.tsx` (enabled only when a workspace project is open). | | `apps/desktop/src/renderer/components/app/ProjectRecoveryScreen.tsx` | Full-project recovery surface for typed open failures, diagnosis, repair progress, next action, and technical details. `ProjectTabHost` in `App.tsx` renders it full-viewport whenever `projectTransitionError` carries a `code` and `rootPath`. | | `apps/desktop/src/renderer/components/app/ProjectTransitionErrorAlert.tsx` | Fallback dismissible banner for project open/switch failures that lack a code/rootPath (un-coded string errors); it renders nothing once a coded error hands the surface to `ProjectRecoveryScreen`. | | `apps/desktop/src/renderer/components/chat/ChatContinuityRecoveryCard.tsx` | In-transcript choices to retry the original thread, rebuild from ADE history, or start a separate chat. `AgentChatMessageList` renders it in place of a plain notice chip when a `system_notice` event's `detail.kind` is `"continuity_recovery"`. | -| `apps/desktop/src/renderer/components/settings/StorageSection.tsx` | Storage dashboard with volume pressure, category totals, cleanup preview/confirmation, and manual history compression. | -| `apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx` | Preview-confirmed cleanup dialog: lists selected removable items with sizes, surfaces blocked paths and reasons, and only enables Remove once a fresh preview is in hand. | -| `apps/desktop/src/renderer/components/settings/storage/storageView.ts` | Pure, DOM-free presentation + policy helpers: category metadata/order/hues, safety labels, and `buildCleanupTarget` / `cleanableEntries` / `groupLaneItems`, which map a snapshot item to a typed `StorageCleanupTarget` (path-derived kind, archived-lane id lookup) so the removable set the UI offers matches what the backend will accept. | -| `apps/desktop/src/shared/types/storage.ts` | Shared storage contracts: `DiskPressureState`/`DiskPressureSnapshot`/`DiskPressureThresholds`, `StorageCategoryId`, `StorageSafety`, `StorageItem`/`StorageCategorySnapshot`/`StorageSnapshot`, and the `StorageCleanupTarget`/`StorageCleanupPreview`/`StorageCleanupResult` DTOs. | +| `apps/desktop/src/renderer/components/settings/StorageSection.tsx` | Storage dashboard: a "Clean up safely" primary action (runs the doctor, falls back to cleanup-by-target on older daemons), the Health & diagnostics strip, category totals with policy chips, a project-database breakdown card with per-row prune/compact actions, the recent-cleanups journal, cleanup preview/confirmation, and manual history compression. | +| `apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx` | The "Health & diagnostics" strip: four tiles — database size (with a journal-fed sparkline + trend arrow), background-service resident memory, slow responses in 24 h (from `getRuntimeHealth`), and last cleanup — plus the overall health chip. Deep-linked as `#diagnostics` from the top-bar load pill. | +| `apps/desktop/src/renderer/components/settings/storage/StorageMaintenanceJournal.tsx` | Collapsible "Recent cleanups" panel rendering the last runs from the maintenance journal, one humanized line per action. | +| `apps/desktop/src/renderer/components/settings/storage/storageUiConstants.ts` | Shared presentational constants (`STORAGE_BRAND`, `PANEL_STYLE`) for the section shell and the split-out diagnostics/journal components, so they share styling without a circular import. | +| `apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx` | Preview-confirmed cleanup dialog: lists selected removable items with sizes, surfaces blocked paths and reasons, and only enables Remove once a fresh preview is in hand. Also hosts the itemized "Clean up safely" plan and its `runMaintenance` path. | +| `apps/desktop/src/renderer/components/settings/storage/storageView.ts` | Pure, DOM-free presentation + policy helpers. Category metadata/order/hues, safety labels, and `buildCleanupTarget` / `cleanableEntries` / `groupLaneItems` map a snapshot item to a typed `StorageCleanupTarget`. The overhaul adds the diagnostics/maintenance view-model: `dbBreakdownRows`, `buildSafeCleanupPlan`, journal/db-size-sparkline/trend helpers, `daemonMemoryBytes`, `healthChip`, `formatSlowActions`, and `categoryPolicyChip` — each degrading to a sensible "not available" value so the UI renders against an older daemon that never sends `extras`. | +| `apps/desktop/src/shared/types/storage.ts` | Shared storage contracts: disk-pressure types, `StorageCategoryId`, `StorageSafety`, `StorageItem`/`StorageCategorySnapshot`/`StorageSnapshot`, and the `StorageCleanupTarget`/`StorageCleanupPreview`/`StorageCleanupResult` DTOs. The overhaul adds the ledger/maintenance surface: `StorageLedgerEntry`/`StoragePolicyClass`, `MaintenanceAction`/`MaintenanceRunReport`/`MaintenanceTrigger`, `DbBreakdownEntry`, `StorageSnapshotExtras` (the optional `extras` on a snapshot), and `RuntimeHealthSnapshot`. | | `apps/desktop/src/shared/types/recovery.ts` | Typed recovery contracts: the `AdeRecoveryErrorCode` union + `toAdeRecoveryErrorCode`, `AdeLastFailureReport`, `ProjectRecoveryDiagnosis`, the ordered `RepairStepId` list + `ProjectRepairReport`, and `mapKvDbOpenErrorCode`. | | `apps/desktop/src/shared/codedError.ts` | `codedError(message, code)`, `encodeCodedErrorMessage`, and the `parseCodedErrorMessage`/`stripElectronErrorWrapper`/`extractCodeFromMessage` decoders that let the renderer recover a `code` through the Electron IPC error-wrapping. Re-exported to the renderer via `apps/desktop/src/renderer/lib/codedError.ts`. | @@ -165,6 +172,108 @@ and modification time did not change. Only a byte-identical verified result is renamed to `.gz` and allowed to replace the original. Any append path first re-inflates the gzip atomically and removes the compressed copy. +### Storage doctor maintenance sweep + +`storageInsightsService` runs a single **storage doctor** sweep that keeps ADE's +footprint bounded without ever touching user data. It is single-flighted (a +concurrent call joins the in-flight run) and fires on three triggers: +`post_boot` (10 minutes after the real daemon instance starts), `daily` (every +24 h thereafter), and `manual` (`runMaintenanceNow()`, wired to the +`storage.runMaintenanceNow` action / IPC and the section's "Clean up safely" +button). Only the real daemon instance — the one constructed with both +`isPathActive` and `diskPressure` — arms the timers; the desktop in-process +fallback instance never schedules maintenance and only acts if +`runMaintenanceNow` is called directly. + +Each run is a fixed sequence of independently try/caught steps (one failing step +never aborts the run), reusing the same validate/preview/cleanup pipeline the +manual flow uses so nothing outside the safe set is ever removed: + +1. Compress inactive chat/terminal history (`fs.transcripts`). +2. Reap stale `safe_to_remove` `ade-*` staging in the system temp root + (`fs.tmp_staging`) and direct children of the project-relative `.ade/tmp` + release-staging dir (`fs.tmp`). +3. Reap obsolete recovery backups (`fs.recovery_backups`), always sparing the + single newest good copy, and the iOS simulator DerivedData cache + (`fs.ios_derived_data`). +4. Invoke the kvDb DB-maintenance hooks: prune `automation_ingress_events`, + `review_run_artifacts`, and `pull_request_snapshots`; compact cr-sqlite + sync bookkeeping; and vacuum when the freelist is fragmented. + +The run is appended to the maintenance journal and summarized in a +`storage.maintenance_completed` log with a `completed` / `partial` / `failed` +outcome. Exactly one deduped `ade_feature_used` (`feature: storage_doctor`) +analytics event is captured at the daemon boundary per completed run, collapsed +to one per 20 h so a daily sweep plus a manual "Clean up now" reads as a single +product event. + +### Storage ledger and policy + +`storageLedger.ts` declares `STORAGE_LEDGER`, the single source of truth for what +every persistent table and directory holds, its privacy class, and how it is +bounded. Privacy classes are `user_data` (never auto-deleted), `derived` +(re-derivable/re-fetchable), and `operational` (bookkeeping). Enforcement is one +of `write_time` (bounded as it is written), `doctor` (swept by the maintenance +run), `both`, or `manual` (only removed on explicit user request). The ledger +is the source the Settings policy chips read (`deriveCategoryPolicyChips`) and +the DB retention numbers derive from the shared `dbMaintenanceApi` constants, so +a policy change updates every enforcement site and the Settings copy in lockstep. + +`LEDGER_LAYOUT_COVERAGE` maps every `ADE_LAYOUT_DEFINITIONS` directory to a +ledger id, or to `null` when the directory is intentionally not storage-managed +(git-tracked scaffold, agent-runtime scratch, or credentials the doctor must +never sweep). A coverage test in `storageInsightsService.test.ts` asserts that +every layout directory appears in the map, so adding a new tracked directory +without declaring its storage policy fails CI. + +### Database maintenance hooks + +`kvDb.ts` attaches a `maintenance` (`DbMaintenanceApi`) handle to the open +database. Every hook is wrapped so any failure logs `db.maintenance_failed` and +returns an `unsupported` skip rather than throwing, and each first checks that +its target table exists: + +- `pruneIngressEvents` — age (7 d) + per-project count (2,000) prune of + `automation_ingress_events`. +- `pruneReviewArtifacts` — delete `review_run_artifacts` older than 30 days. +- `prunePrSnapshots` — delete `pull_request_snapshots` not updated in 60 days. +- `compactCrsqlTombstones` — rebuild the `operations` CRR table to shed + cr-sqlite clock/pks shadow rows, then vacuum. **Only runs when the project has + zero sync peers** (`options.hasSyncPeers`, defaulting conservatively to + "assume peers"); otherwise it returns a `has_peers` skip and touches nothing, + because compacting shared change-tracking state mid-sync is unsafe. +- `vacuumIfFragmented(threshold)` — when the freelist fraction exceeds the + threshold, a one-time full `VACUUM` rebuilds the file and activates + `auto_vacuum = INCREMENTAL`; every later sweep then reclaims the freelist in + bounded `incremental_vacuum` chunks (≤ 25 × 2,000 pages per call) so a blocking + full VACUUM never runs again. Each call finishes with a `wal_checkpoint(TRUNCATE)` + and reports bytes reclaimed from the on-disk footprint (`.db` + `-wal` + `-shm`). + +The database is opened in WAL mode with `synchronous = NORMAL` +(`journal_mode = WAL` set explicitly at open in `openRawDatabase`). + +### Maintenance journal, diagnostics, and runtime health + +Every doctor run appends a `MaintenanceRunReport` (trigger, per-action results, +bytes reclaimed, and the post-run DB size) to `storage-doctor-journal.json` +under `.ade/cache`, capped at the last 30 runs and written atomically. The +journal is a plain rebuildable file — no DB, no CRR — so a corrupt or missing +journal degrades to empty. + +The Settings > Storage snapshot carries an optional `extras` block built from the +journal and a `dbstat` scan: the project-database breakdown (webhooks, sync +bookkeeping, review artifacts, PR cache, core — `dbstat` is treated as optional +and degrades to no breakdown when the SQLite build lacks it), the recent-runs +journal, the derived per-category policy chips, and a safe-reclaimable byte +estimate. The renderer's "Health & diagnostics" strip renders a DB-size +sparkline/trend from the journal, the background service's resident memory, the +last-cleanup headline, and a **slow-responses (24 h)** tile. That last figure +comes from `LocalRuntimeConnectionPool.getRuntimeHealth()` (surfaced by the +`ade.app.getRuntimeHealth` IPC), a rolling, age- and count-bounded window of +daemon action calls that took over 500 ms or errored — the same calls that emit +`local_runtime.action_slow`. The top-bar resource-pressure "load pill" deep-links +into this strip via `#/settings?tab=storage#diagnostics`. + ### Backup and retention bounds | State | Bound | Retention rule | @@ -173,6 +282,10 @@ re-inflates the gzip atomically and removes the compressed copy. | Thread-pointer ledger | 64 KiB | Compacts to the newest valid record per session; a malformed tail is ignored. | | Last-failure reports | Current + 1 previous | Repeated same-signature failures increment the current report; a changed signature rotates current to previous. | | Database migration backup | 1 | `.pre-crsqlite-w1.bak` is created once and only with sufficient headroom. | +| Automation ingress events | 7 days / 2,000 rows per project | Age-pruned at write time and by the doctor; the newest 2,000 non-`dispatched` rows per project are kept regardless of age (dispatched rows are age-pruned only). Raw webhook payloads are no longer persisted. | +| Review artifacts | 30 days | `review_run_artifacts` older than the cutoff are deleted (re-derivable from a fresh review). | +| PR snapshots | 60 days | `pull_request_snapshots` not updated within the window are deleted (re-fetchable from GitHub). | +| Storage-doctor journal | 30 runs | `storage-doctor-journal.json` keeps the newest 30 `MaintenanceRunReport`s; older runs drop off on the next append. | | launchd logs | 10 MiB threshold × 2 streams | `launchd.err.log` and `launchd.out.log` each keep a 1 MiB `.1` tail, then copytruncate the live file to zero. | | Desktop JSONL logs | 10 MiB × 2 generations | Current log plus one `.1` rotation; the older rotation is replaced. | | Transparent compressed history read | 256 MiB decompressed | Larger inputs are rejected rather than expanded in memory. | @@ -196,3 +309,13 @@ re-inflates the gzip atomically and removes the compressed copy. - Writers append to plain history. If a `.gz` exists, reinflate atomically before append; never append a second plain stream beside the compressed generation. +- cr-sqlite compaction (`compactCrsqlTombstones`) is only safe with **zero sync + peers**. Leave `hasSyncPeers` conservative — the default assumes peers, and a + paired project's sync-bookkeeping row stays "waiting to compact" rather than + offering an action that would be peer-blocked. +- The DB retention numbers live once in `state/dbMaintenanceApi.ts`. The ingress + writer, the kvDb hooks, and the storage ledger all import them; never re-hard-code + a cutoff in one enforcement site. +- `dbstat` is a compile-time-optional virtual table. The breakdown scan and the + vacuum path degrade gracefully when the SQLite build lacks it — do not assume + it is present. From dfac5af42ef5e46fc897bd5fa8a3551be6ccc0b5 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:39:00 -0400 Subject: [PATCH 04/10] =?UTF-8?q?ship:=20iteration=201=20=E2=80=94=20prese?= =?UTF-8?q?rve=20dispatched=20ingress=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/services/state/dbMaintenanceApi.ts | 4 +- .../state/kvDb.rebuildRecovery.test.ts | 40 +++++++++++++++++++ apps/desktop/src/main/services/state/kvDb.ts | 6 +-- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/main/services/state/dbMaintenanceApi.ts b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts index bcd356949..d7379d88f 100644 --- a/apps/desktop/src/main/services/state/dbMaintenanceApi.ts +++ b/apps/desktop/src/main/services/state/dbMaintenanceApi.ts @@ -10,7 +10,7 @@ const DAY_MS = 24 * 60 * 60 * 1_000; /** Automation ingress events older than this are pruned (write-time + doctor). */ export const INGRESS_EVENT_RETENTION_MS = 7 * DAY_MS; -/** Newest ingress rows kept per project regardless of age (count cap). */ +/** Newest non-dispatched ingress rows kept per project after age pruning. */ export const INGRESS_EVENT_MAX_ROWS_PER_PROJECT = 2_000; /** Review artifacts older than this (in days) are deleted. */ export const REVIEW_ARTIFACT_RETENTION_DAYS = 30; @@ -24,7 +24,7 @@ export type DbMaintenanceResult = { }; export interface DbMaintenanceApi { - /** Age+count prune of automation_ingress_events (7d / 2,000 per project). */ + /** Age-prune all ingress rows, then cap non-dispatched rows at 2,000 per project. */ pruneIngressEvents(): DbMaintenanceResult; /** Delete review_run_artifacts rows older than 30 days. */ pruneReviewArtifacts(): DbMaintenanceResult; diff --git a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts index 3d8a84142..65cf0472e 100644 --- a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts +++ b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts @@ -405,6 +405,46 @@ describe("kvDb storage maintenance", () => { expect(db.get<{ synchronous: number }>("pragma synchronous")?.synchronous).toBe(1); }); + it("caps eligible ingress rows without deleting dispatched history", async () => { + const dbPath = makeDbPath(); + const db = await openKvDb(dbPath, createLogger()); + closeLater(db); + db.run(` + create table automation_ingress_events ( + id text primary key, + project_id text not null, + status text not null, + received_at text not null + ) + `); + + const receivedAt = new Date(Date.now() - 24 * 60 * 60 * 1_000).toISOString(); + db.run( + "insert into automation_ingress_events(id, project_id, status, received_at) values (?, ?, ?, ?)", + ["dispatched-oldest", "project-1", "dispatched", receivedAt], + ); + db.run("begin immediate"); + for (let index = 0; index < 2_001; index += 1) { + db.run( + "insert into automation_ingress_events(id, project_id, status, received_at) values (?, ?, ?, ?)", + [`ignored-${index}`, "project-1", "ignored", receivedAt], + ); + } + db.run("commit"); + + expect(db.maintenance?.pruneIngressEvents()).toEqual({ + itemsAffected: 1, + bytesReclaimed: 0, + skippedReason: null, + }); + expect(db.get<{ count: number }>( + "select count(*) as count from automation_ingress_events where status = 'ignored'", + )?.count).toBe(2_000); + expect(db.get<{ count: number }>( + "select count(*) as count from automation_ingress_events where id = 'dispatched-oldest'", + )?.count).toBe(1); + }); + it("reclaims a fragmented file, activates incremental auto-vacuum, and skips a healthy file", async () => { const dbPath = makeDbPath(); const logs: LogEntry[] = []; diff --git a/apps/desktop/src/main/services/state/kvDb.ts b/apps/desktop/src/main/services/state/kvDb.ts index 2f89facb4..d71969024 100644 --- a/apps/desktop/src/main/services/state/kvDb.ts +++ b/apps/desktop/src/main/services/state/kvDb.ts @@ -3758,7 +3758,7 @@ export async function openKvDb( const maintenance: DbMaintenanceApi = { pruneIngressEvents: () => runMaintenanceSafely("pruneIngressEvents", () => { if (!rawHasTable(db, "automation_ingress_events")) return unsupportedMaintenanceResult(); - const cutoff = new Date(Date.now() - 7 * 24 * 60 * 60 * 1_000).toISOString(); + const cutoff = new Date(Date.now() - INGRESS_EVENT_RETENTION_MS).toISOString(); let itemsAffected = runStatement( db, "delete from automation_ingress_events where received_at < ?", @@ -3775,9 +3775,9 @@ export async function openKvDb( where rowid in ( select rowid from automation_ingress_events - where project_id = ? + where project_id = ? and status != 'dispatched' order by received_at desc, rowid desc - limit -1 offset 2000 + limit -1 offset ${INGRESS_EVENT_MAX_ROWS_PER_PROJECT} )`, [project.project_id], ).changes; From 36d2cb2b7003874f2ae84de0db1788b7b07a798c Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:25:44 -0400 Subject: [PATCH 05/10] =?UTF-8?q?ship:=20iteration=202=20=E2=80=94=20fix?= =?UTF-8?q?=20ingress=20replay=20and=20cleanup=20estimate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../automations/automationService.test.ts | 13 +--- .../services/automations/automationService.ts | 76 +++++++++++-------- .../storage/storageInsightsService.test.ts | 31 +++++++- .../storage/storageInsightsService.ts | 14 +++- 4 files changed, 88 insertions(+), 46 deletions(-) diff --git a/apps/desktop/src/main/services/automations/automationService.test.ts b/apps/desktop/src/main/services/automations/automationService.test.ts index e91d3b27b..29f32750a 100644 --- a/apps/desktop/src/main/services/automations/automationService.test.ts +++ b/apps/desktop/src/main/services/automations/automationService.test.ts @@ -2493,35 +2493,30 @@ describe("automation ingress storage bounds", () => { "select count(*) as count from automation_ingress_events where raw_payload_json is not null", ))[0]?.count).toBe(0); - const original = await service.dispatchIngressTrigger({ + const unexpiredReplay = await service.dispatchIngressTrigger({ source: "github-relay", eventKey: "storm-9999", triggerType: "github.issue_opened", rawPayload: { replay: "within-window" }, }); - expect(original).not.toBeNull(); + expect(unexpiredReplay).not.toBeNull(); const storedOriginal = mapExecRows(raw.exec( "select id from automation_ingress_events where event_key = 'storm-9999'", )); expect(storedOriginal).toHaveLength(1); - expect(original?.id).toBe(storedOriginal[0]?.id); + expect(unexpiredReplay?.id).toBe(storedOriginal[0]?.id); raw.run( "update automation_ingress_events set received_at = ? where event_key = 'storm-9999'", [new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000).toISOString()], ); - await service.dispatchIngressTrigger({ - source: "github-relay", - eventKey: "ttl-prune-trigger", - triggerType: "github.issue_opened", - }); const replayed = await service.dispatchIngressTrigger({ source: "github-relay", eventKey: "storm-9999", triggerType: "github.issue_opened", rawPayload: { replay: "after-window" }, }); - expect(replayed?.id).not.toBe(original?.id); + expect(replayed?.id).not.toBe(unexpiredReplay?.id); expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where raw_payload_json is not null", ))[0]?.count).toBe(0); diff --git a/apps/desktop/src/main/services/automations/automationService.ts b/apps/desktop/src/main/services/automations/automationService.ts index a9781623a..3af1b0787 100644 --- a/apps/desktop/src/main/services/automations/automationService.ts +++ b/apps/desktop/src/main/services/automations/automationService.ts @@ -1137,14 +1137,17 @@ export function createAutomationService({ [tableName], )); - const pruneIngressEventsForProject = (targetProjectId: string, referenceTime = new Date()): void => { + const pruneExpiredIngressEventsForProject = (targetProjectId: string, referenceTime = new Date()): void => { const cutoff = new Date(referenceTime.getTime() - INGRESS_EVENT_RETENTION_MS).toISOString(); db.run( "delete from automation_ingress_events where project_id = ? and received_at < ?", [targetProjectId, cutoff], ); + }; + + const pruneIngressEventOverflowForProject = (targetProjectId: string): void => { // Dispatched events are exempt from the count cap (they may still be read - // by run detail / the UI); they remain subject to the age prune above, so + // by run detail / the UI); they remain subject to the age prune, so // they still expire at the retention horizon. db.run( `delete from automation_ingress_events @@ -1159,6 +1162,11 @@ export function createAutomationService({ ); }; + const pruneIngressEventsForProject = (targetProjectId: string, referenceTime = new Date()): void => { + pruneExpiredIngressEventsForProject(targetProjectId, referenceTime); + pruneIngressEventOverflowForProject(targetProjectId); + }; + const reclaimLegacyIngressPayloads = (): void => { const hasPayloads = db.get<{ present: number }>( "select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1", @@ -3392,42 +3400,47 @@ export function createAutomationService({ }): Promise => { const eventKey = args.eventKey.trim(); if (!eventKey.length) return null; - const existing = db.get( - ` - select - id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, - cursor, received_at - from automation_ingress_events - where project_id = ? and source = ? and event_key = ? - limit 1 - `, - [projectId, args.source, eventKey] - ); - if (existing) return toIngressEvent(existing); - const eventId = randomUUID(); const receivedAt = nowIso(); + let existing: AutomationIngressEventRow | null = null; db.run("begin immediate"); try { - db.run( + // Prune before dedupe so an event can be replayed after the retention + // window. Keeping both operations in this write transaction also avoids + // racing the unique (project, source, event_key) index. + pruneExpiredIngressEventsForProject(projectId, new Date(receivedAt)); + existing = db.get( ` - insert into automation_ingress_events( - id, project_id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, cursor, raw_payload_json, received_at - ) values (?, ?, ?, ?, '[]', ?, ?, 'received', ?, null, ?, null, ?) + select + id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, + cursor, received_at + from automation_ingress_events + where project_id = ? and source = ? and event_key = ? + limit 1 `, - [ - eventId, - projectId, - args.source, - eventKey, - args.triggerType, - args.eventName ?? null, - args.summary ?? null, - args.cursor ?? null, - receivedAt, - ] + [projectId, args.source, eventKey] ); - pruneIngressEventsForProject(projectId, new Date(receivedAt)); + if (!existing) { + db.run( + ` + insert into automation_ingress_events( + id, project_id, source, event_key, automation_ids_json, trigger_type, event_name, status, summary, error_message, cursor, raw_payload_json, received_at + ) values (?, ?, ?, ?, '[]', ?, ?, 'received', ?, null, ?, null, ?) + `, + [ + eventId, + projectId, + args.source, + eventKey, + args.triggerType, + args.eventName ?? null, + args.summary ?? null, + args.cursor ?? null, + receivedAt, + ] + ); + pruneIngressEventOverflowForProject(projectId); + } db.run("commit"); } catch (error) { try { @@ -3437,6 +3450,7 @@ export function createAutomationService({ } throw error; } + if (existing) return toIngressEvent(existing); if (args.cursor) upsertIngressCursor(args.source, args.cursor); const trigger: TriggerContext = { diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index ef84bf889..4ca8d33bb 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -570,7 +570,21 @@ describe("storageInsightsService", () => { service.dispose(); }); - it("populates snapshot extras with db breakdown, policy chips, and safe-reclaimable bytes", async () => { + it("keeps database-only cleanup actionable by counting prunable db bytes", async () => { + const snapshot = await createStorageInsightsService({ + projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), + }).getSnapshot(); + const extras = snapshot.extras!; + const prunableDbBytes = extras.dbBreakdown.reduce( + (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), + 0, + ); + + expect(prunableDbBytes).toBeGreaterThan(0); + expect(extras.safeReclaimableBytes).toBe(prunableDbBytes); + }); + + it("adds prunable db bytes to filesystem cleanup without double-counting", async () => { writeSized(path.join(projectRoot, ".ade", "cache", "browser-observations", "c.bin"), 100); writeSized(path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData", "m.o"), 200); const snapshot = await createStorageInsightsService({ @@ -581,8 +595,19 @@ describe("storageInsightsService", () => { expect(extras.maintenance.journal).toEqual([]); expect(extras.maintenance.lastRun).toBeNull(); expect(extras.policyChips.chats_history).toBe("Compressed after 14 days"); - // safe_to_remove caches (browser-observations 100 + DerivedData 200) are counted. - expect(extras.safeReclaimableBytes).toBeGreaterThanOrEqual(300); + const prunableDbBytes = extras.dbBreakdown.reduce( + (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), + 0, + ); + const safeFilesystemBytes = snapshot.categories.reduce( + (categorySum, category) => categorySum + category.items.reduce( + (itemSum, item) => itemSum + (item.safety === "safe_to_remove" ? item.bytes : 0), + 0, + ), + 0, + ); + expect(safeFilesystemBytes).toBe(300); + expect(extras.safeReclaimableBytes).toBe(prunableDbBytes + safeFilesystemBytes); if (extras.dbBreakdown.length > 0) { expect(extras.dbBreakdown.some((entry) => entry.category === "core")).toBe(true); } diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index 868960219..2ac61cabd 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -598,15 +598,23 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti buildCategory("recovery_backups", categoryItems.get("recovery_backups") ?? [], "review_first", state), buildCategory("database", categoryItems.get("database") ?? [], "protected", state), ]; - let safeReclaimableBytes = compressibleBytes; + const journal = readMaintenanceJournal(journalPath); + const dbBreakdown = computeDbBreakdown(deriveSyncBookkeepingAction(journal)); + // Database storage is represented only by protected filesystem items above, + // so adding the prunable logical categories here cannot double-count it. + // Deliberately exclude sync bookkeeping: even when its display state is + // "compactable", the maintenance hook can still be peer-gated at run time. + let safeReclaimableBytes = compressibleBytes + dbBreakdown.reduce( + (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), + 0, + ); for (const items of categoryItems.values()) { for (const item of items) { if (item.safety === "safe_to_remove") safeReclaimableBytes += item.bytes; } } - const journal = readMaintenanceJournal(journalPath); const extras: StorageSnapshotExtras = { - dbBreakdown: computeDbBreakdown(deriveSyncBookkeepingAction(journal)), + dbBreakdown, maintenance: { lastRun: journal[0] ?? null, journal }, safeReclaimableBytes, policyChips: deriveCategoryPolicyChips(), From af512803b54cb74258a69d6f33dc7b3654e6cf3e Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:56:33 -0400 Subject: [PATCH 06/10] =?UTF-8?q?ship:=20iteration=203=20=E2=80=94=20align?= =?UTF-8?q?=20safe=20cleanup=20estimate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/storageInsightsService.test.ts | 24 +++++++++++++++---- .../storage/storageInsightsService.ts | 7 +++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 4ca8d33bb..d2bc56039 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -584,9 +584,17 @@ describe("storageInsightsService", () => { expect(extras.safeReclaimableBytes).toBe(prunableDbBytes); }); - it("adds prunable db bytes to filesystem cleanup without double-counting", async () => { + it("adds presented filesystem cleanup to prunable db bytes without counting recovery backups", async () => { writeSized(path.join(projectRoot, ".ade", "cache", "browser-observations", "c.bin"), 100); writeSized(path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData", "m.o"), 200); + const newerBackup = path.join(projectRoot, ".ade", "ade.db.recovery-newer.bak"); + const obsoleteBackup = path.join(projectRoot, ".ade", "ade.db.recovery-obsolete.bak"); + writeSized(newerBackup, 400); + writeSized(obsoleteBackup, 500); + const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60_000); + const nineDaysAgo = new Date(Date.now() - 9 * 24 * 60 * 60_000); + fs.utimesSync(newerBackup, eightDaysAgo, eightDaysAgo); + fs.utimesSync(obsoleteBackup, nineDaysAgo, nineDaysAgo); const snapshot = await createStorageInsightsService({ projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), }).getSnapshot(); @@ -599,15 +607,21 @@ describe("storageInsightsService", () => { (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), 0, ); - const safeFilesystemBytes = snapshot.categories.reduce( - (categorySum, category) => categorySum + category.items.reduce( + const presentedFilesystemBytes = snapshot.categories + .filter((category) => category.id === "caches" || category.id === "build_release") + .reduce((categorySum, category) => categorySum + category.items.reduce( (itemSum, item) => itemSum + (item.safety === "safe_to_remove" ? item.bytes : 0), 0, ), 0, ); - expect(safeFilesystemBytes).toBe(300); - expect(extras.safeReclaimableBytes).toBe(prunableDbBytes + safeFilesystemBytes); + const recoveryBackups = snapshot.categories.find((category) => category.id === "recovery_backups")!; + expect(recoveryBackups.items.find((item) => item.path === obsoleteBackup)).toMatchObject({ + bytes: 500, + safety: "safe_to_remove", + }); + expect(presentedFilesystemBytes).toBe(300); + expect(extras.safeReclaimableBytes).toBe(prunableDbBytes + presentedFilesystemBytes); if (extras.dbBreakdown.length > 0) { expect(extras.dbBreakdown.some((entry) => entry.category === "core")).toBe(true); } diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index 2ac61cabd..34c1aaa10 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -608,7 +608,12 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), 0, ); - for (const items of categoryItems.values()) { + // The primary cleanup dialog only itemizes rebuildable caches and + // build/release staging. Recovery backups remain an independent, + // review-first surface even when the doctor can safely reap obsolete copies, + // so they must not inflate the amount this CTA promises to reclaim. + for (const categoryId of ["build_release", "caches"] as const) { + const items = categoryItems.get(categoryId) ?? []; for (const item of items) { if (item.safety === "safe_to_remove") safeReclaimableBytes += item.bytes; } From 1068ce77f46100a066c38add828f9b7dd74fa3bd Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:39:07 -0400 Subject: [PATCH 07/10] =?UTF-8?q?ship:=20iteration=204=20=E2=80=94=20prote?= =?UTF-8?q?ct=20active=20iOS=20build=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ade-cli/src/bootstrap.ts | 3 +- apps/desktop/src/main/main.ts | 3 +- .../services/ios/iosSimulatorService.test.ts | 51 ++++++++++++ .../main/services/ios/iosSimulatorService.ts | 42 ++++++---- .../storage/storageInsightsService.test.ts | 79 ++++++++++++++++++- .../storage/storageInsightsService.ts | 52 ++++++++++-- 6 files changed, 204 insertions(+), 26 deletions(-) diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index 7c2fb3651..7fd4be718 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -1503,7 +1503,8 @@ export async function createAdeRuntime(args: { diskPressure: diskPressureMonitor, isPathActive: (filePath) => Boolean(agentChatService?.isTranscriptPathActive(filePath)) - || ptyService.isTranscriptPathActive(filePath), + || ptyService.isTranscriptPathActive(filePath) + || Boolean(iosSimulatorService?.isBuildPathActive(filePath)), projectId, // One bounded `ade_feature_used` per completed maintenance run at the daemon // boundary (deduped to 20 h by the service). diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 0515a9ef6..ae218ef32 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -3586,7 +3586,8 @@ app.whenReady().then(async () => { diskPressure: diskPressureMonitor, isPathActive: (filePath) => agentChatService.isTranscriptPathActive(filePath) - || ptyService.isTranscriptPathActive(filePath), + || ptyService.isTranscriptPathActive(filePath) + || iosSimulatorService.isBuildPathActive(filePath), projectId, captureAnalytics: (input) => { productAnalyticsService.capture(input); diff --git a/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts b/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts index cfe57a699..19c39e399 100644 --- a/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts +++ b/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts @@ -438,6 +438,57 @@ describe("iosSimulatorService Simulator.app launch visibility", () => { } }); + it("reports the managed DerivedData path active only while xcodebuild owns it", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const projectRoot = fs.mkdtempSync(`${os.tmpdir()}/ade-ios-build-activity-`); + writeMinimalXcodeProject(projectRoot, "Prox"); + let markBuildStarted!: () => void; + const buildStarted = new Promise((resolve) => { markBuildStarted = resolve; }); + let releaseBuild!: () => void; + const buildRelease = new Promise((resolve) => { releaseBuild = resolve; }); + const runMock = vi.fn(async (command: string, commandArgs: string[]) => { + if (command === "ps") return { stdout: "", stderr: "" }; + if (command === "/usr/bin/xcode-select") return { stdout: "", stderr: "" }; + if (command === "xcodebuild" && commandArgs[0] === "-version") { + return { stdout: "Xcode 26.3\nBuild version 17C52\n", stderr: "" }; + } + if (command === "xcodebuild") { + markBuildStarted(); + await buildRelease; + throw new Error("test build stopped"); + } + if (command === "xcrun" && commandArgs.join(" ") === "simctl list devices available --json") { + return { stdout: simulatorDevicesJson, stderr: "" }; + } + if (command === "xcrun" && commandArgs[1] === "bootstatus") return { stdout: "", stderr: "" }; + if (command === "xcrun" && commandArgs[1] === "listapps") return { stdout: "", stderr: "" }; + return { stdout: "", stderr: "" }; + }); + const restoreHooks = __testSetIosSimulatorProcessHooks({ + run: runMock, + commandExists: () => true, + }); + const service = createIosSimulatorService({ projectRoot, logger: noopLogger }); + const derivedDataPath = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); + + try { + const launchPromise = service.launch({ projectRoot, build: true }); + await buildStarted; + expect(service.isBuildPathActive(derivedDataPath)).toBe(true); + expect(service.isBuildPathActive(path.join(projectRoot, ".ade", "cache", "other"))).toBe(false); + + releaseBuild(); + await expect(launchPromise).rejects.toThrow(/test build stopped/); + expect(service.isBuildPathActive(derivedDataPath)).toBe(false); + } finally { + releaseBuild(); + service.dispose(); + fs.rmSync(projectRoot, { recursive: true, force: true }); + restoreHooks(); + platformSpy.mockRestore(); + } + }); + it("opens Simulator.app by default during launch", async () => { const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); const runMock = vi.fn(async (command: string, commandArgs: string[]) => { diff --git a/apps/desktop/src/main/services/ios/iosSimulatorService.ts b/apps/desktop/src/main/services/ios/iosSimulatorService.ts index c0b8c89d8..3a7c49e72 100644 --- a/apps/desktop/src/main/services/ios/iosSimulatorService.ts +++ b/apps/desktop/src/main/services/ios/iosSimulatorService.ts @@ -1749,6 +1749,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { }; let controlQueue: Promise = Promise.resolve(); let activeLaunchId: string | null = null; + const activeBuildDataPaths = new Map(); let disposed = false; let xcodeMcpBridge: XcodeMcpBridge | null = null; const tempFiles = new Set(); @@ -2920,7 +2921,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { }; const buildProjectApp = async (target: ResolvedLaunchTarget, device: IosSimulatorDevice, projectRoot: string): Promise => { - const derivedDataPath = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); + const derivedDataPath = path.resolve(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); if (!target.projectPath || !target.scheme) { return target.appBundlePath; } @@ -2935,21 +2936,28 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { derivedDataPath, }); try { - await run("xcodebuild", [ - "-project", - relativeProjectPath, - "-scheme", - target.scheme, - "-configuration", - "Debug", - "-sdk", - "iphonesimulator", - "-destination", - `platform=iOS Simulator,id=${device.udid}`, - "-derivedDataPath", - derivedDataPath, - "build", - ], { cwd: projectRoot, timeoutMs: 10 * 60_000 }); + activeBuildDataPaths.set(derivedDataPath, (activeBuildDataPaths.get(derivedDataPath) ?? 0) + 1); + try { + await run("xcodebuild", [ + "-project", + relativeProjectPath, + "-scheme", + target.scheme, + "-configuration", + "Debug", + "-sdk", + "iphonesimulator", + "-destination", + `platform=iOS Simulator,id=${device.udid}`, + "-derivedDataPath", + derivedDataPath, + "build", + ], { cwd: projectRoot, timeoutMs: 10 * 60_000 }); + } finally { + const remaining = (activeBuildDataPaths.get(derivedDataPath) ?? 1) - 1; + if (remaining > 0) activeBuildDataPaths.set(derivedDataPath, remaining); + else activeBuildDataPaths.delete(derivedDataPath); + } } catch (error) { const formatted = formatXcodeBuildFailure(error, { projectPath: relativeProjectPath, @@ -3742,12 +3750,14 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { swipe: drag, selectPoint, getLastSelectedItem: () => lastSelectedItem, + isBuildPathActive: (candidatePath: string) => activeBuildDataPaths.has(path.resolve(candidatePath)), dispose: () => { disposed = true; stopCompanion(); setStreamStopped(null); activeSession = null; activeLaunchId = null; + activeBuildDataPaths.clear(); cleanupTempFiles(); toolAvailabilityCache.clear(); if (xcodeMcpBridge) { diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index d2bc56039..742ba6a33 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -410,6 +410,67 @@ describe("storageInsightsService", () => { expect(fs.existsSync(staging)).toBe(false); }); + it("blocks active iOS DerivedData at execution time and removes stale data once inactive", async () => { + const derivedData = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); + const objectFile = path.join(derivedData, "module.o"); + writeSized(objectFile, 37); + let buildActive = false; + const service = createStorageInsightsService({ + projectRoot, + adeHome, + db, + logger, + isPathActive: (candidatePath) => buildActive && path.resolve(candidatePath) === derivedData, + stagingTmpDir: path.join(adeHome, "no-real-staging"), + }); + const target = { kind: "rebuildable_cache" as const, path: derivedData }; + + const recentSnapshot = await service.getSnapshot({ forceRefresh: true }); + expect(recentSnapshot.categories.find((category) => category.id === "build_release")?.items) + .toContainEqual(expect.objectContaining({ path: derivedData, safety: "review_first" })); + + const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60_000); + fs.utimesSync(objectFile, eightDaysAgo, eightDaysAgo); + fs.utimesSync(derivedData, eightDaysAgo, eightDaysAgo); + buildActive = true; + const activeSnapshot = await service.getSnapshot({ forceRefresh: true }); + expect(activeSnapshot.categories.find((category) => category.id === "build_release")?.items) + .toContainEqual(expect.objectContaining({ path: derivedData, safety: "review_first" })); + expect((await service.cleanupPreview([target])).blocked).toEqual([ + { path: derivedData, reason: "An iOS build is currently using this data." }, + ]); + const iosCacheRoot = path.dirname(derivedData); + expect((await service.cleanupPreview([{ kind: "rebuildable_cache", path: iosCacheRoot }])).blocked) + .toEqual([{ path: iosCacheRoot, reason: "An iOS build is currently using this data." }]); + + buildActive = false; + const inactiveSnapshot = await service.getSnapshot({ forceRefresh: true }); + expect(inactiveSnapshot.categories.find((category) => category.id === "build_release")?.items) + .toContainEqual(expect.objectContaining({ path: derivedData, safety: "safe_to_remove" })); + const preview = await service.cleanupPreview([target]); + expect(preview.items).toHaveLength(1); + + // A build that starts after confirmation is caught by the execution-time + // activity check, so a valid stale preview cannot delete its DerivedData. + buildActive = true; + const blocked = await service.cleanup([target], { preview }); + expect(blocked.removed).toEqual([]); + expect(blocked.failed).toEqual([ + { path: derivedData, reason: "An iOS build is currently using this data." }, + ]); + expect(fs.existsSync(derivedData)).toBe(true); + + buildActive = false; + const inactivePreview = await service.cleanupPreview([target]); + const removed = await service.cleanup([target], { preview: inactivePreview }); + expect(removed).toEqual({ + removed: [{ path: derivedData, bytes: 37 }], + failed: [], + freedBytes: 37, + }); + expect(fs.existsSync(derivedData)).toBe(false); + }); + const normalDiskPressure = { getSnapshot: () => ({ state: "normal" as const, @@ -458,7 +519,10 @@ describe("storageInsightsService", () => { // iOS DerivedData build cache. const derived = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); - writeSized(path.join(derived, "module.o"), 70); + const derivedObject = path.join(derived, "module.o"); + writeSized(derivedObject, 70); + fs.utimesSync(derivedObject, eightDaysAgo, eightDaysAgo); + fs.utimesSync(derived, eightDaysAgo, eightDaysAgo); const service = createStorageInsightsService({ projectRoot, adeHome, db, logger, @@ -586,17 +650,26 @@ describe("storageInsightsService", () => { it("adds presented filesystem cleanup to prunable db bytes without counting recovery backups", async () => { writeSized(path.join(projectRoot, ".ade", "cache", "browser-observations", "c.bin"), 100); - writeSized(path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData", "m.o"), 200); + const derivedData = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); + const derivedObject = path.join(derivedData, "m.o"); + writeSized(derivedObject, 200); const newerBackup = path.join(projectRoot, ".ade", "ade.db.recovery-newer.bak"); const obsoleteBackup = path.join(projectRoot, ".ade", "ade.db.recovery-obsolete.bak"); writeSized(newerBackup, 400); writeSized(obsoleteBackup, 500); const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60_000); const nineDaysAgo = new Date(Date.now() - 9 * 24 * 60 * 60_000); + fs.utimesSync(derivedObject, eightDaysAgo, eightDaysAgo); + fs.utimesSync(derivedData, eightDaysAgo, eightDaysAgo); fs.utimesSync(newerBackup, eightDaysAgo, eightDaysAgo); fs.utimesSync(obsoleteBackup, nineDaysAgo, nineDaysAgo); const snapshot = await createStorageInsightsService({ - projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), + projectRoot, + adeHome, + db, + logger, + isPathActive: () => false, + stagingTmpDir: path.join(adeHome, "no-real-staging"), }).getSnapshot(); const extras = snapshot.extras!; expect(Array.isArray(extras.dbBreakdown)).toBe(true); diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index 34c1aaa10..fa3a23813 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -278,6 +278,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti const cacheTtlMs = options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS; const scanEntryLimit = options.scanEntryLimit ?? DEFAULT_SCAN_ENTRY_LIMIT; const scanBudgetMs = options.scanBudgetMs ?? DEFAULT_SCAN_BUDGET_MS; + const isPathActive = options.isPathActive ?? (() => true); let cachedSnapshot: { value: StorageSnapshot; createdAt: number } | null = null; const compressionRoots: CompressionRoots = [ { path: layout.chatTranscriptsDir, kind: "chat_transcript" }, @@ -289,7 +290,7 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti logger: options.logger, diskPressure: options.diskPressure, // Without the runtime's in-memory ownership signal, compression is unsafe. - isPathActive: options.isPathActive ?? (() => true), + isPathActive, }); // `.ade/tmp` is a project-relative release-staging root (distinct from the // `.ade/cache/tmp` layout dir) written by the /release skill and never cleaned @@ -303,6 +304,17 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti let firstSweepTimer: ReturnType | null = null; let dailySweepTimer: ReturnType | null = null; + const overlapsIosDerivedData = (targetPath: string): boolean => + isSameOrWithin(targetPath, iosDerivedDataDir) || isSameOrWithin(iosDerivedDataDir, targetPath); + + const iosDerivedDataBlockReason = (lastModifiedMs: number | null): string | null => { + if (isPathActive(iosDerivedDataDir)) return "An iOS build is currently using this data."; + if (lastModifiedMs != null && Date.now() - lastModifiedMs <= STALE_AGE_MS) { + return "This iOS build data was used recently and may still be in use."; + } + return null; + }; + const runCompressionSweep = (opts?: { maxFiles?: number }): Promise => { if (sweepFlight) return sweepFlight; sweepFlight = compressor.runIdleSweep(compressionRoots, opts).finally(() => { @@ -503,15 +515,23 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti add("build_release", entry?.item); } const derivedData = iosDerivedDataDir; - add("build_release", (await makeItem({ + const derivedDataEntry = await makeItem({ category: "build_release", base: layout.adeDir, path: derivedData, label: "iOS build data", - safety: "safe_to_remove", + safety: "review_first", state, - detail: "Recreated the next time you build", - }))?.item); + }); + if (derivedDataEntry) { + const lastModifiedMs = derivedDataEntry.item.lastModifiedAt + ? Date.parse(derivedDataEntry.item.lastModifiedAt) + : null; + const blockedReason = iosDerivedDataBlockReason(lastModifiedMs); + derivedDataEntry.item.safety = blockedReason ? "review_first" : "safe_to_remove"; + derivedDataEntry.item.detail = blockedReason ?? "Recreated the next time you build"; + add("build_release", derivedDataEntry.item); + } const cacheNames = await readdirOrEmpty(layout.cacheDir); for (const name of cacheNames) { @@ -713,6 +733,15 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti } const size = await walkPath(targetPath, null); + if ( + target.kind === "rebuildable_cache" + && overlapsIosDerivedData(targetPath) + && await lstatOrNull(iosDerivedDataDir) + ) { + const derivedDataSize = await walkPath(iosDerivedDataDir, null); + const blockedReason = iosDerivedDataBlockReason(derivedDataSize.lastModifiedMs); + if (blockedReason) return { valid: null, reason: blockedReason }; + } const identity = [stat.dev, stat.ino, stat.size, stat.mtimeMs, size.bytes, size.lastModifiedMs ?? 0].join(":"); return { valid: { target, path: targetPath, bytes: size.bytes, label, identity } }; }; @@ -787,6 +816,19 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti failed.push({ path: rawPath, reason: "This item changed after the preview. Preview it again before removing it." }); continue; } + if ( + target.kind === "rebuildable_cache" + && overlapsIosDerivedData(checked.valid.path) + && await lstatOrNull(iosDerivedDataDir) + ) { + const blockedReason = iosDerivedDataBlockReason( + (await walkPath(iosDerivedDataDir, null)).lastModifiedMs, + ); + if (blockedReason) { + failed.push({ path: rawPath, reason: blockedReason }); + continue; + } + } try { if (target.kind === "orphaned_worktree" || target.kind === "archived_lane_worktree") { await removeWorktree(checked.valid.path); From 1c84009d4d1460bb7b13b3b85f03eb4d98d1310d Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Fri, 17 Jul 2026 23:52:21 -0400 Subject: [PATCH 08/10] =?UTF-8?q?ship:=20iteration=203=20=E2=80=94=20addre?= =?UTF-8?q?ss=2016=20review=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ade-cli/src/bootstrap.ts | 13 ++- apps/desktop/src/main/main.ts | 16 +++- .../automations/automationService.test.ts | 43 +++++++++- .../services/automations/automationService.ts | 36 ++++---- .../state/kvDb.rebuildRecovery.test.ts | 45 +++++++++- apps/desktop/src/main/services/state/kvDb.ts | 10 ++- .../services/storage/storageDbBreakdown.ts | 16 ++-- .../storage/storageInsightsService.test.ts | 70 ++++++++++++++-- .../storage/storageInsightsService.ts | 45 ++++++---- .../storage/storageMaintenanceJournal.ts | 32 ++++++- .../settings/StorageSection.test.tsx | 83 +++++++++++++++++-- .../components/settings/StorageSection.tsx | 11 ++- .../settings/storage/StorageCleanupDialog.tsx | 25 +++--- .../settings/storage/StorageDiagnostics.tsx | 17 +++- .../settings/storage/storageView.test.ts | 49 +++++++++-- .../settings/storage/storageView.ts | 60 +++++++++++++- 16 files changed, 476 insertions(+), 95 deletions(-) diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index 7fd4be718..96d7c34f2 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -504,9 +504,15 @@ export async function createAdeRuntime(args: { const diskPressureMonitor = createDiskPressureMonitor({ roots: [projectRoot, resolveMachineAdeLayout().adeDir], }); + // A sync-enabled runtime must prove zero connected peers before CRR + // compaction. Runtimes without sync have no peer transport and can safely + // report zero immediately. + let liveSyncPeerCount: number | null = resolvedArgs.syncRuntime?.enabled ? null : 0; let db: AdeDb; try { - db = await openKvDb(paths.dbPath, logger); + db = await openKvDb(paths.dbPath, logger, { + hasSyncPeers: () => liveSyncPeerCount !== 0, + }); } catch (error) { const code = mapKvDbOpenErrorCode(classifySqliteOpenError(error)); const detail = error instanceof Error ? error.message : String(error); @@ -1639,7 +1645,10 @@ export async function createAdeRuntime(args: { getModelPickerStore: () => getSharedModelPickerStore(db), cloudRelayStore, syncTunnelClientService, - onStatusChanged: (snapshot) => pushEvent("runtime", { type: "sync-status", snapshot }), + onStatusChanged: (snapshot) => { + liveSyncPeerCount = snapshot.connectedPeers.length; + pushEvent("runtime", { type: "sync-status", snapshot }); + }, }); syncServiceForPtyEvents = syncService; } diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index ae218ef32..c7f5fb826 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -2171,8 +2171,14 @@ app.whenReady().then(async () => { } }; + // Fail closed until sync publishes its first status, then keep the DB + // compaction gate synchronized with the live connected-peer count. + let liveSyncPeerCount: number | null = null; + let syncServiceRef: ReturnType | null = null; const db = await measureProjectInitStep("db_open", () => - openKvDb(adePaths.dbPath, logger), + openKvDb(adePaths.dbPath, logger, { + hasSyncPeers: () => liveSyncPeerCount !== 0, + }), ); const keybindingsService = createKeybindingsService({ db }); const agentToolsService = createAgentToolsService({ logger }); @@ -2906,7 +2912,6 @@ app.whenReady().then(async () => { }); }; - let syncServiceRef: ReturnType | null = null; const ptyBackend = process.env.ADE_DISABLE_SUPERVISED_PTY_HOST === "1" ? null : createSupervisedPtyLoader({ logger }); @@ -3656,6 +3661,7 @@ app.whenReady().then(async () => { projectScaffoldService.listMyGitHubRepos(input), }, onStatusChanged: (snapshot) => { + liveSyncPeerCount = snapshot.connectedPeers.length; const normalizedProjectRoot = normalizeProjectRoot(projectRoot); if (mobileSyncSelectedRoot == null && snapshot.connectedPeers.length > 0) { mobileSyncSelectedRoot = normalizedProjectRoot; @@ -4319,7 +4325,11 @@ app.whenReady().then(async () => { const logger = createFileLogger(path.join(adePaths.logsDir, "main.jsonl")); const project = toProjectInfo(projectRoot, baseRef); const runtimeProject = await localRuntimePool.ensureProject(projectRoot); - const db = await openKvDb(adePaths.dbPath, logger); + const db = await openKvDb(adePaths.dbPath, logger, { + // The machine runtime owns sync and the real storage doctor in this mode; + // this dormant fallback has no authoritative local peer signal. + hasSyncPeers: () => true, + }); const shellContext = createDormantProjectContext(projectRoot, { enableUsageTracking: false }); const storageInsightsService = createStorageInsightsService({ projectRoot, diff --git a/apps/desktop/src/main/services/automations/automationService.test.ts b/apps/desktop/src/main/services/automations/automationService.test.ts index 29f32750a..ae4037210 100644 --- a/apps/desktop/src/main/services/automations/automationService.test.ts +++ b/apps/desktop/src/main/services/automations/automationService.test.ts @@ -2470,7 +2470,7 @@ describe("automation ingress storage bounds", () => { }); } - it("keeps only 2,000 slim rows during an insert storm and preserves the seven-day dedup window", async () => { + it("caps never-dispatched rows during an insert storm and preserves dispatched dedupe history", async () => { const { db, raw } = createInMemoryAdeDb(); const service = createIngressService(db); @@ -2521,15 +2521,20 @@ describe("automation ingress storage bounds", () => { "select count(*) as count from automation_ingress_events where raw_payload_json is not null", ))[0]?.count).toBe(0); - // Dispatched rows are exempt from the count cap; an equally-old ignored - // row beyond the newest 2,000 is not. Both are within the 7-day window so - // the age prune leaves them; only the count cap acts. + // Dispatched and failed-after-dispatch rows are exempt from the count cap; + // an equally-old ignored row beyond the newest 2,000 is not. All are + // within the 7-day window so only the count cap acts. const fiveDaysAgo = new Date(Date.now() - 5 * 24 * 60 * 60 * 1_000).toISOString(); raw.run( `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) values ('keep-dispatched', 'proj', 'github-relay', 'keep-dispatched', '[]', 'github.issue_opened', 'dispatched', null, ?)`, [fiveDaysAgo], ); + raw.run( + `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) + values ('keep-failed', 'proj', 'github-relay', 'keep-failed', '[]', 'github.issue_opened', 'failed', null, ?)`, + [fiveDaysAgo], + ); raw.run( `insert into automation_ingress_events(id, project_id, source, event_key, automation_ids_json, trigger_type, status, raw_payload_json, received_at) values ('drop-ignored', 'proj', 'github-relay', 'drop-ignored', '[]', 'github.issue_opened', 'ignored', null, ?)`, @@ -2544,6 +2549,9 @@ describe("automation ingress storage bounds", () => { expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where event_key = 'keep-dispatched'", ))[0]?.count).toBe(1); + expect(mapExecRows(raw.exec( + "select count(*) as count from automation_ingress_events where event_key = 'keep-failed'", + ))[0]?.count).toBe(1); expect(mapExecRows(raw.exec( "select count(*) as count from automation_ingress_events where event_key = 'drop-ignored'", ))[0]?.count).toBe(0); @@ -2552,6 +2560,33 @@ describe("automation ingress storage bounds", () => { } }, 120_000); + it("runs startup retention even when no legacy ingress payloads remain", () => { + const { db, raw } = createInMemoryAdeDb(); + raw.run("create table review_run_artifacts(id text primary key, created_at text not null)"); + raw.run("create table pull_request_snapshots(pr_id text primary key, updated_at text not null)"); + const oldIngress = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000).toISOString(); + const oldReview = new Date(Date.now() - 31 * 24 * 60 * 60 * 1_000).toISOString(); + const oldSnapshot = new Date(Date.now() - 61 * 24 * 60 * 60 * 1_000).toISOString(); + raw.run( + `insert into automation_ingress_events( + id, project_id, source, event_key, automation_ids_json, trigger_type, + status, raw_payload_json, received_at + ) values ('stale-ingress', 'proj', 'github-relay', 'stale-ingress', '[]', 'github.issue_opened', 'ignored', null, ?)`, + [oldIngress], + ); + raw.run("insert into review_run_artifacts values ('stale-review', ?)", [oldReview]); + raw.run("insert into pull_request_snapshots values ('stale-pr', ?)", [oldSnapshot]); + + const service = createIngressService(db); + try { + expect(mapExecRows(raw.exec("select id from automation_ingress_events"))).toEqual([]); + expect(mapExecRows(raw.exec("select id from review_run_artifacts"))).toEqual([]); + expect(mapExecRows(raw.exec("select pr_id from pull_request_snapshots"))).toEqual([]); + } finally { + service.dispose(); + } + }); + it("reclaims legacy payloads and local caches once, in bounded chunks", () => { const { db, raw } = createInMemoryAdeDb(); raw.run("create table review_run_artifacts(id text primary key, created_at text not null)"); diff --git a/apps/desktop/src/main/services/automations/automationService.ts b/apps/desktop/src/main/services/automations/automationService.ts index 3af1b0787..46e605b98 100644 --- a/apps/desktop/src/main/services/automations/automationService.ts +++ b/apps/desktop/src/main/services/automations/automationService.ts @@ -1146,15 +1146,15 @@ export function createAutomationService({ }; const pruneIngressEventOverflowForProject = (targetProjectId: string): void => { - // Dispatched events are exempt from the count cap (they may still be read - // by run detail / the UI); they remain subject to the age prune, so - // they still expire at the retention horizon. + // Rows that reached dispatch are exempt from the count cap. A failed row + // was previously dispatched and must keep deduping webhook redelivery even + // when a later action failed; both statuses remain subject to the age prune. db.run( `delete from automation_ingress_events where rowid in ( select rowid from automation_ingress_events - where project_id = ? and status != 'dispatched' + where project_id = ? and status not in ('dispatched', 'failed') order by received_at desc, rowid desc limit -1 offset ${INGRESS_EVENT_MAX_ROWS_PER_PROJECT} )`, @@ -1171,15 +1171,15 @@ export function createAutomationService({ const hasPayloads = db.get<{ present: number }>( "select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1", ); - if (!hasPayloads) return; - - const payloadStats = db.get<{ rows_cleared: number; bytes_cleared: number | null }>( - `select count(*) as rows_cleared, coalesce(sum(length(raw_payload_json)), 0) as bytes_cleared - from automation_ingress_events - where raw_payload_json is not null`, - ); - - while (db.get("select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1")) { + const payloadStats = hasPayloads + ? db.get<{ rows_cleared: number; bytes_cleared: number | null }>( + `select count(*) as rows_cleared, coalesce(sum(length(raw_payload_json)), 0) as bytes_cleared + from automation_ingress_events + where raw_payload_json is not null`, + ) + : null; + + while (hasPayloads && db.get("select 1 as present from automation_ingress_events where raw_payload_json is not null limit 1")) { db.run("begin immediate"); try { db.run( @@ -1224,10 +1224,12 @@ export function createAutomationService({ ); } - logger.info("automations.ingress_payload_reclaim", { - rowsCleared: Number(payloadStats?.rows_cleared ?? 0), - bytesCleared: Number(payloadStats?.bytes_cleared ?? 0), - }); + if (hasPayloads) { + logger.info("automations.ingress_payload_reclaim", { + rowsCleared: Number(payloadStats?.rows_cleared ?? 0), + bytesCleared: Number(payloadStats?.bytes_cleared ?? 0), + }); + } }; const ensureSchema = () => { diff --git a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts index 65cf0472e..04df1f75e 100644 --- a/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts +++ b/apps/desktop/src/main/services/state/kvDb.rebuildRecovery.test.ts @@ -13,6 +13,10 @@ import { recoverInterruptedTableRebuilds, type TableRebuildPlan, } from "./kvDb"; +import { + PR_SNAPSHOT_RETENTION_DAYS, + REVIEW_ARTIFACT_RETENTION_DAYS, +} from "./dbMaintenanceApi"; const require = createRequire(import.meta.url); const { DatabaseSync } = require("node:sqlite") as { @@ -405,7 +409,7 @@ describe("kvDb storage maintenance", () => { expect(db.get<{ synchronous: number }>("pragma synchronous")?.synchronous).toBe(1); }); - it("caps eligible ingress rows without deleting dispatched history", async () => { + it("caps never-dispatched ingress rows without deleting dispatched or failed history", async () => { const dbPath = makeDbPath(); const db = await openKvDb(dbPath, createLogger()); closeLater(db); @@ -423,6 +427,10 @@ describe("kvDb storage maintenance", () => { "insert into automation_ingress_events(id, project_id, status, received_at) values (?, ?, ?, ?)", ["dispatched-oldest", "project-1", "dispatched", receivedAt], ); + db.run( + "insert into automation_ingress_events(id, project_id, status, received_at) values (?, ?, ?, ?)", + ["failed-oldest", "project-1", "failed", receivedAt], + ); db.run("begin immediate"); for (let index = 0; index < 2_001; index += 1) { db.run( @@ -443,6 +451,36 @@ describe("kvDb storage maintenance", () => { expect(db.get<{ count: number }>( "select count(*) as count from automation_ingress_events where id = 'dispatched-oldest'", )?.count).toBe(1); + expect(db.get<{ count: number }>( + "select count(*) as count from automation_ingress_events where id = 'failed-oldest'", + )?.count).toBe(1); + }); + + it("applies the shared review-artifact and PR-snapshot retention windows", async () => { + const db = await openKvDb(makeDbPath(), createLogger()); + closeLater(db); + db.run("pragma foreign_keys = off"); + const now = Date.now(); + const reviewOld = new Date(now - (REVIEW_ARTIFACT_RETENTION_DAYS + 1) * 24 * 60 * 60 * 1_000).toISOString(); + const reviewRecent = new Date(now - (REVIEW_ARTIFACT_RETENTION_DAYS - 1) * 24 * 60 * 60 * 1_000).toISOString(); + const prOld = new Date(now - (PR_SNAPSHOT_RETENTION_DAYS + 1) * 24 * 60 * 60 * 1_000).toISOString(); + const prRecent = new Date(now - (PR_SNAPSHOT_RETENTION_DAYS - 1) * 24 * 60 * 60 * 1_000).toISOString(); + db.run( + `insert into review_run_artifacts(id, run_id, artifact_type, title, mime_type, created_at) + values ('review-old', 'missing-run', 'summary', 'Old', 'text/plain', ?), + ('review-recent', 'missing-run', 'summary', 'Recent', 'text/plain', ?)`, + [reviewOld, reviewRecent], + ); + db.run( + `insert into pull_request_snapshots(pr_id, updated_at) + values ('pr-old', ?), ('pr-recent', ?)`, + [prOld, prRecent], + ); + + expect(db.maintenance?.pruneReviewArtifacts().itemsAffected).toBe(1); + expect(db.maintenance?.prunePrSnapshots().itemsAffected).toBe(1); + expect(db.all<{ id: string }>("select id from review_run_artifacts order by id")).toEqual([{ id: "review-recent" }]); + expect(db.all<{ pr_id: string }>("select pr_id from pull_request_snapshots order by pr_id")).toEqual([{ pr_id: "pr-recent" }]); }); it("reclaims a fragmented file, activates incremental auto-vacuum, and skips a healthy file", async () => { @@ -537,13 +575,16 @@ describe("kvDb storage maintenance", () => { it("gates CRR tombstone compaction on peer state", async () => { const pairedPath = makeDbPath(); - const paired = await openKvDb(pairedPath, createLogger(), { hasSyncPeers: () => true }); + let hasSyncPeers = true; + const paired = await openKvDb(pairedPath, createLogger(), { hasSyncPeers: () => hasSyncPeers }); closeLater(paired); expect(paired.maintenance?.compactCrsqlTombstones()).toEqual({ itemsAffected: 0, bytesReclaimed: 0, skippedReason: "has_peers", }); + hasSyncPeers = false; + expect(paired.maintenance?.compactCrsqlTombstones()?.skippedReason).not.toBe("has_peers"); }); it.skipIf(process.platform === "linux")("compacts CRR tombstones with no peers and preserves operations byte-for-byte", async () => { diff --git a/apps/desktop/src/main/services/state/kvDb.ts b/apps/desktop/src/main/services/state/kvDb.ts index d71969024..3835f7682 100644 --- a/apps/desktop/src/main/services/state/kvDb.ts +++ b/apps/desktop/src/main/services/state/kvDb.ts @@ -3775,7 +3775,7 @@ export async function openKvDb( where rowid in ( select rowid from automation_ingress_events - where project_id = ? and status != 'dispatched' + where project_id = ? and status not in ('dispatched', 'failed') order by received_at desc, rowid desc limit -1 offset ${INGRESS_EVENT_MAX_ROWS_PER_PROJECT} )`, @@ -3786,7 +3786,9 @@ export async function openKvDb( }), pruneReviewArtifacts: () => runMaintenanceSafely("pruneReviewArtifacts", () => { if (!rawHasTable(db, "review_run_artifacts")) return unsupportedMaintenanceResult(); - const cutoff = new Date(Date.now() - 30 * 24 * 60 * 60 * 1_000).toISOString(); + const cutoff = new Date( + Date.now() - REVIEW_ARTIFACT_RETENTION_DAYS * 24 * 60 * 60 * 1_000, + ).toISOString(); const itemsAffected = runStatement( db, "delete from review_run_artifacts where created_at < ?", @@ -3796,7 +3798,9 @@ export async function openKvDb( }), prunePrSnapshots: () => runMaintenanceSafely("prunePrSnapshots", () => { if (!rawHasTable(db, "pull_request_snapshots")) return unsupportedMaintenanceResult(); - const cutoff = new Date(Date.now() - 60 * 24 * 60 * 60 * 1_000).toISOString(); + const cutoff = new Date( + Date.now() - PR_SNAPSHOT_RETENTION_DAYS * 24 * 60 * 60 * 1_000, + ).toISOString(); const itemsAffected = runStatement( db, "delete from pull_request_snapshots where updated_at < ?", diff --git a/apps/desktop/src/main/services/storage/storageDbBreakdown.ts b/apps/desktop/src/main/services/storage/storageDbBreakdown.ts index 26f4c73d5..9a9a8334a 100644 --- a/apps/desktop/src/main/services/storage/storageDbBreakdown.ts +++ b/apps/desktop/src/main/services/storage/storageDbBreakdown.ts @@ -60,16 +60,18 @@ export function mapDbBreakdown( /** * Sync-bookkeeping compaction state, derived without any new seam. We only * surface an actionable "Compact now" ("compactable") once the journal proves - * the most recent run's cr-sqlite compaction actually executed without a - * has_peers skip. With no journal yet — or no compact record in the latest run - * — there is no positive evidence that compaction is safe, so we default to - * "compaction_pending" ("Waiting to compact") rather than offer an action that - * might turn out to be peer-blocked. A latest run peer-blocked stays pending. + * the most recent run's cr-sqlite compaction actually completed successfully. + * With no journal yet, no compact record, an error, or any skip reason, there + * is no positive evidence that the action can run, so we stay pending. */ export function deriveSyncBookkeepingAction( journal: readonly MaintenanceRunReport[], ): DbBreakdownEntry["action"] { - const lastCompact = journal[0]?.actions.find((action) => action.ledgerId === "db.operations_crsql"); + const lastCompact = journal[0]?.actions.find( + (action) => action.ledgerId === "db.operations_crsql" && action.kind === "compact", + ); if (!lastCompact) return "compaction_pending"; - return lastCompact.skippedReason === "has_peers" ? "compaction_pending" : "compactable"; + return !lastCompact.error && lastCompact.skippedReason == null + ? "compactable" + : "compaction_pending"; } diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 742ba6a33..868ca7fd3 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -6,6 +6,7 @@ import type { MaintenanceRunReport, StorageCleanupPreview, StorageCleanupTarget import { openKvDb, type AdeDb } from "../state/kvDb"; import { createStorageInsightsService } from "./storageInsightsService"; import { classifyDbTable, deriveSyncBookkeepingAction, mapDbBreakdown } from "./storageDbBreakdown"; +import { isMaintenanceReport, readMaintenanceJournal } from "./storageMaintenanceJournal"; import { ADE_LAYOUT_DEFINITIONS } from "../../../shared/adeLayout"; import { deriveCategoryPolicyChips, @@ -634,7 +635,7 @@ describe("storageInsightsService", () => { service.dispose(); }); - it("keeps database-only cleanup actionable by counting prunable db bytes", async () => { + it("does not promise dbstat table totals as reclaimable bytes", async () => { const snapshot = await createStorageInsightsService({ projectRoot, adeHome, db, logger, stagingTmpDir: path.join(adeHome, "no-real-staging"), }).getSnapshot(); @@ -645,24 +646,27 @@ describe("storageInsightsService", () => { ); expect(prunableDbBytes).toBeGreaterThan(0); - expect(extras.safeReclaimableBytes).toBe(prunableDbBytes); + expect(extras.safeReclaimableBytes).toBe(0); }); - it("adds presented filesystem cleanup to prunable db bytes without counting recovery backups", async () => { + it("counts only filesystem targets accepted by the cleanup validator", async () => { writeSized(path.join(projectRoot, ".ade", "cache", "browser-observations", "c.bin"), 100); const derivedData = path.join(projectRoot, ".ade", "cache", "ios-simulator", "DerivedData"); const derivedObject = path.join(derivedData, "m.o"); writeSized(derivedObject, 200); const newerBackup = path.join(projectRoot, ".ade", "ade.db.recovery-newer.bak"); const obsoleteBackup = path.join(projectRoot, ".ade", "ade.db.recovery-obsolete.bak"); + const appUpdateStaging = path.join(adeHome, "runtime", "updates", "stale-update.zip"); writeSized(newerBackup, 400); writeSized(obsoleteBackup, 500); + writeSized(appUpdateStaging, 600); const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60_000); const nineDaysAgo = new Date(Date.now() - 9 * 24 * 60 * 60_000); fs.utimesSync(derivedObject, eightDaysAgo, eightDaysAgo); fs.utimesSync(derivedData, eightDaysAgo, eightDaysAgo); fs.utimesSync(newerBackup, eightDaysAgo, eightDaysAgo); fs.utimesSync(obsoleteBackup, nineDaysAgo, nineDaysAgo); + fs.utimesSync(appUpdateStaging, nineDaysAgo, nineDaysAgo); const snapshot = await createStorageInsightsService({ projectRoot, adeHome, @@ -693,8 +697,13 @@ describe("storageInsightsService", () => { bytes: 500, safety: "safe_to_remove", }); - expect(presentedFilesystemBytes).toBe(300); - expect(extras.safeReclaimableBytes).toBe(prunableDbBytes + presentedFilesystemBytes); + expect(presentedFilesystemBytes).toBe(900); + expect(snapshot.categories.find((category) => category.id === "caches")?.items).toContainEqual( + expect.objectContaining({ path: appUpdateStaging, safety: "safe_to_remove" }), + ); + // Project cache + stale DerivedData are valid targets (300 bytes). App + // update staging is outside the project cache validator and is omitted. + expect(extras.safeReclaimableBytes).toBe(300); if (extras.dbBreakdown.length > 0) { expect(extras.dbBreakdown.some((entry) => entry.category === "core")).toBe(true); } @@ -751,10 +760,16 @@ describe("storageInsightsService", () => { expect(deriveSyncBookkeepingAction([ run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: null, itemsAffected: 3 }), ])).toBe("compactable"); - // Hook absent last run (unsupported) → still compactable, never lies "pending". + // Unsupported or failed work is not evidence that compaction can run. expect(deriveSyncBookkeepingAction([ run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: "unsupported" }), - ])).toBe("compactable"); + ])).toBe("compaction_pending"); + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "compact", error: "compaction failed" }), + ])).toBe("compaction_pending"); + expect(deriveSyncBookkeepingAction([ + run({ ledgerId: "db.operations_crsql", kind: "vacuum", skippedReason: null }), + ])).toBe("compaction_pending"); // Newest run wins even if an older run was peer-blocked. expect(deriveSyncBookkeepingAction([ run({ ledgerId: "db.operations_crsql", kind: "compact", skippedReason: null }), @@ -763,6 +778,47 @@ describe("storageInsightsService", () => { }); }); +describe("storageMaintenanceJournal", () => { + const validReport: MaintenanceRunReport = { + startedAt: "2026-07-17T00:00:00.000Z", + finishedAt: "2026-07-17T00:00:01.000Z", + trigger: "manual", + actions: [{ + ledgerId: "db.operations_crsql", + kind: "compact", + itemsAffected: 2, + bytesReclaimed: 64, + durationMs: 8, + skippedReason: null, + error: null, + }], + reclaimedBytes: 64, + dbSizeBytes: null, + }; + + it("accepts complete reports and rejects malformed nested fields", () => { + expect(isMaintenanceReport(validReport)).toBe(true); + expect(isMaintenanceReport({ ...validReport, trigger: "sometimes" })).toBe(false); + expect(isMaintenanceReport({ ...validReport, actions: [null] })).toBe(false); + expect(isMaintenanceReport({ ...validReport, actions: [{ ...validReport.actions[0], durationMs: -1 }] })).toBe(false); + expect(isMaintenanceReport({ ...validReport, dbSizeBytes: undefined })).toBe(false); + }); + + it("filters malformed entries before downstream journal consumers see them", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-storage-journal-")); + const journalPath = path.join(root, "journal.json"); + try { + fs.writeFileSync(journalPath, JSON.stringify([ + { ...validReport, actions: [null] }, + validReport, + ])); + expect(readMaintenanceJournal(journalPath)).toEqual([validReport]); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); + describe("storageLedger", () => { it("declares a well-formed, unique policy for every entry", () => { const ids = new Set(); diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.ts b/apps/desktop/src/main/services/storage/storageInsightsService.ts index fa3a23813..33ae54296 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.ts @@ -620,24 +620,37 @@ export function createStorageInsightsService(options: StorageInsightsServiceOpti ]; const journal = readMaintenanceJournal(journalPath); const dbBreakdown = computeDbBreakdown(deriveSyncBookkeepingAction(journal)); - // Database storage is represented only by protected filesystem items above, - // so adding the prunable logical categories here cannot double-count it. - // Deliberately exclude sync bookkeeping: even when its display state is - // "compactable", the maintenance hook can still be peer-gated at run time. - let safeReclaimableBytes = compressibleBytes + dbBreakdown.reduce( - (sum, entry) => sum + (entry.action === "prunable" ? entry.bytes : 0), - 0, - ); - // The primary cleanup dialog only itemizes rebuildable caches and - // build/release staging. Recovery backups remain an independent, - // review-first surface even when the doctor can safely reap obsolete copies, - // so they must not inflate the amount this CTA promises to reclaim. - for (const categoryId of ["build_release", "caches"] as const) { - const items = categoryItems.get(categoryId) ?? []; - for (const item of items) { - if (item.safety === "safe_to_remove") safeReclaimableBytes += item.bytes; + // Estimate only work the same backend validator will authorize. Raw + // compressible bytes are not predicted savings, and dbstat table totals do + // not identify the expired rows (or the bytes a later vacuum can return), + // so neither is defensible enough for the primary cleanup promise. + const candidateTargets = new Map(); + for (const category of categories) { + if (category.id !== "build_release" && category.id !== "caches") continue; + for (const item of category.items) { + if (item.safety !== "safe_to_remove") continue; + const itemPath = path.resolve(item.path); + if ( + (isDirectChild(stagingTmpRoot, itemPath) && /^ade-/.test(path.basename(itemPath))) + || isDirectChild(adeTmpDir, itemPath) + ) { + candidateTargets.set(itemPath, { kind: "stale_tmp_staging", path: itemPath }); + } else if (isWithin(layout.cacheDir, itemPath)) { + candidateTargets.set(itemPath, { kind: "rebuildable_cache", path: itemPath }); + } } } + // Prefer an accepted ancestor over descendants so nested cache entries can + // never inflate the estimate by counting the same bytes twice. + const acceptedPaths: string[] = []; + let safeReclaimableBytes = 0; + for (const target of [...candidateTargets.values()].sort((left, right) => left.path.length - right.path.length)) { + if (acceptedPaths.some((acceptedPath) => isSameOrWithin(acceptedPath, target.path))) continue; + const checked = await validateTarget(target); + if (!checked.valid) continue; + acceptedPaths.push(checked.valid.path); + safeReclaimableBytes += checked.valid.bytes; + } const extras: StorageSnapshotExtras = { dbBreakdown, maintenance: { lastRun: journal[0] ?? null, journal }, diff --git a/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts index b8687d639..699edfbf0 100644 --- a/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts +++ b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts @@ -10,14 +10,44 @@ import type { MaintenanceRunReport } from "../../../shared/types/storage"; export const MAINTENANCE_JOURNAL_MAX_RUNS = 30; +const MAINTENANCE_TRIGGERS = new Set(["daily", "post_boot", "manual", "post_migration"]); +const MAINTENANCE_ACTION_KINDS = new Set(["prune", "compress", "delete", "vacuum", "compact", "checkpoint"]); + +function isFiniteNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function isNullableString(value: unknown): value is string | null | undefined { + return value == null || typeof value === "string"; +} + +function isMaintenanceAction(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const action = value as Record; + return typeof action.ledgerId === "string" + && action.ledgerId.length > 0 + && typeof action.kind === "string" + && MAINTENANCE_ACTION_KINDS.has(action.kind) + && isFiniteNonNegativeNumber(action.itemsAffected) + && isFiniteNonNegativeNumber(action.bytesReclaimed) + && isFiniteNonNegativeNumber(action.durationMs) + && isNullableString(action.skippedReason) + && isNullableString(action.error); +} + export function isMaintenanceReport(value: unknown): value is MaintenanceRunReport { if (!value || typeof value !== "object") return false; const report = value as Record; return typeof report.startedAt === "string" + && report.startedAt.length > 0 && typeof report.finishedAt === "string" + && report.finishedAt.length > 0 && typeof report.trigger === "string" + && MAINTENANCE_TRIGGERS.has(report.trigger) && Array.isArray(report.actions) - && typeof report.reclaimedBytes === "number"; + && report.actions.every(isMaintenanceAction) + && isFiniteNonNegativeNumber(report.reclaimedBytes) + && (report.dbSizeBytes === null || isFiniteNonNegativeNumber(report.dbSizeBytes)); } export function readMaintenanceJournal(journalPath: string): MaintenanceRunReport[] { diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx index 102baed57..3481f65d3 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx @@ -82,11 +82,12 @@ function makeSnapshot(): StorageSnapshot { }, { id: "recovery_backups", - bytes: 80 * 1024 ** 2, + bytes: 140 * 1024 ** 2, fileCount: 2, safety: "review_first", items: [ - { id: "r1", label: "Recovery backup", path: `${PROJECT}/.ade/ade.db.bak-2026-07-01`, bytes: 80 * 1024 ** 2, fileCount: 1, lastModifiedAt: new Date().toISOString(), safety: "review_first" }, + { id: "r-old", label: "Obsolete recovery backup", path: `${PROJECT}/.ade/ade.db.bak-2026-06-01`, bytes: 60 * 1024 ** 2, fileCount: 1, lastModifiedAt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString(), safety: "safe_to_remove" }, + { id: "r-new", label: "Newest recovery backup", path: `${PROJECT}/.ade/ade.db.bak-2026-07-01`, bytes: 80 * 1024 ** 2, fileCount: 1, lastModifiedAt: new Date().toISOString(), safety: "review_first" }, ], }, { @@ -184,7 +185,13 @@ function makeUsage(): AppResourceUsageSnapshot { } as AppResourceUsageSnapshot; } -function installAdeMock(options: { withCompress?: boolean; withExtras?: boolean; withApp?: boolean } = {}) { +function installAdeMock(options: { + withCompress?: boolean; + withExtras?: boolean; + withApp?: boolean; + maintenanceReport?: MaintenanceRunReport; + resourceUsage?: AppResourceUsageSnapshot; +} = {}) { const cleanupPreview = vi.fn( async (targets: StorageCleanupTarget[]): Promise => ({ items: targets.map((target) => ({ path: target.path, bytes: 1.5 * 1024 ** 3, label: "Old feature" })), @@ -202,7 +209,7 @@ function installAdeMock(options: { withCompress?: boolean; withExtras?: boolean; ); const compressNow = vi.fn(async () => ({ filesCompressed: 12, savedBytes: 300 * 1024 ** 2 })); const runMaintenanceNow = vi.fn( - async (): Promise => ({ + async (): Promise => options.maintenanceReport ?? ({ startedAt: new Date().toISOString(), finishedAt: new Date().toISOString(), trigger: "manual", @@ -216,7 +223,7 @@ function installAdeMock(options: { withCompress?: boolean; withExtras?: boolean; const getRuntimeHealth = vi.fn( async (): Promise => ({ slowActions24h: 3, slowActionP95Ms: 5_200, sampledAt: new Date().toISOString() }), ); - const getResourceUsage = vi.fn(async () => makeUsage()); + const getResourceUsage = vi.fn(async () => options.resourceUsage ?? makeUsage()); const storage: Record = { getSnapshot: vi.fn(async () => (options.withExtras ? makeSnapshotWithExtras() : makeSnapshot())), @@ -357,11 +364,16 @@ describe("StorageSection", () => { fireEvent.click(primary); const dialog = await screen.findByRole("dialog"); + expect(within(dialog).getByText("iOS build data")).toBeTruthy(); + expect(within(dialog).queryByText("npm")).toBeNull(); + expect(within(dialog).getByText("Obsolete recovery backup")).toBeTruthy(); + expect(within(dialog).queryByText("Newest recovery backup")).toBeNull(); + expect(within(dialog).getAllByText(/newest (recovery )?backup/i).length).toBeGreaterThan(0); const confirm = await within(dialog).findByRole("button", { name: /clean up safely/i }); fireEvent.click(confirm); await waitFor(() => expect(runMaintenanceNow).toHaveBeenCalledTimes(1)); - expect(await within(dialog).findByText(/Freed 481 MB/)).toBeTruthy(); + expect(await within(dialog).findByText(/Reclaimed 481 MB/)).toBeTruthy(); }); it("shows the database breakdown and runs maintenance from an inline action", async () => { @@ -380,6 +392,47 @@ describe("StorageSection", () => { await waitFor(() => expect(runMaintenanceNow).toHaveBeenCalledTimes(1)); }); + it("reports a zero-byte partial maintenance failure instead of calling storage tidy", async () => { + const failedReport: MaintenanceRunReport = { + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + trigger: "manual", + actions: [ + { ledgerId: "db.operations_crsql", kind: "compact", itemsAffected: 0, bytesReclaimed: 0, durationMs: 3, error: "database busy" }, + ], + reclaimedBytes: 0, + dbSizeBytes: 30 * MB, + }; + const { runMaintenanceNow } = installAdeMock({ withExtras: true, maintenanceReport: failedReport }); + render(); + + fireEvent.click(await screen.findByRole("button", { name: /compact now/i })); + await waitFor(() => expect(runMaintenanceNow).toHaveBeenCalledTimes(1)); + expect(await screen.findByText("Some cleanup steps couldn't finish")).toBeTruthy(); + expect(screen.queryByText("Storage is already tidy")).toBeNull(); + }); + + it("shows a partial maintenance failure in the safe-cleanup dialog", async () => { + const failedReport: MaintenanceRunReport = { + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + trigger: "manual", + actions: [ + { ledgerId: "fs.tmp", kind: "delete", itemsAffected: 0, bytesReclaimed: 0, durationMs: 3, error: "files in use" }, + ], + reclaimedBytes: 0, + dbSizeBytes: 30 * MB, + }; + installAdeMock({ withExtras: true, maintenanceReport: failedReport }); + render(); + + fireEvent.click(await screen.findByRole("button", { name: /clean up safely/i })); + const dialog = await screen.findByRole("dialog"); + fireEvent.click(await within(dialog).findByRole("button", { name: /clean up safely/i })); + expect(await within(dialog).findByText("Some cleanup steps couldn't finish.")).toBeTruthy(); + expect(within(dialog).queryByText("Storage is already tidy.")).toBeNull(); + }); + it("renders diagnostics from resource usage and the maintenance journal", async () => { installAdeMock({ withExtras: true }); render(); @@ -397,6 +450,24 @@ describe("StorageSection", () => { expect(await screen.findByText(/reclaimed 450 MB/)).toBeTruthy(); }); + it("does not label unavailable resource-pressure data as healthy", async () => { + const unavailableUsage = { + ...makeUsage(), + cpuPercent: null, + mainCpuPercent: null, + rendererCpuPercent: null, + ptyCpuPercent: null, + memoryMB: null, + freeMemoryMB: null, + } as AppResourceUsageSnapshot; + const { getResourceUsage } = installAdeMock({ withExtras: true, resourceUsage: unavailableUsage }); + render(); + + await waitFor(() => expect(getResourceUsage).toHaveBeenCalledTimes(1)); + await screen.findByText("Health & diagnostics"); + expect(screen.queryByText("Healthy")).toBeNull(); + }); + it("keeps plain language across the diagnostics and database surfaces", async () => { installAdeMock({ withExtras: true }); const { container } = render(); diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.tsx index 6d255ccbb..01f97e7b7 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.tsx @@ -56,6 +56,7 @@ import { formatApproxBytes, formatBytes, groupLaneItems, + maintenanceOutcome, safeReclaimableBytes, type Trend, } from "./storage/storageView"; @@ -738,8 +739,7 @@ export function StorageSection() { setMaintenanceBusy(true); try { const report = await runMaintenanceNow(); - const reclaimed = typeof report?.reclaimedBytes === "number" && Number.isFinite(report.reclaimedBytes) ? report.reclaimedBytes : 0; - showToast(reclaimed > 0 ? `Reclaimed ${formatBytes(reclaimed)}` : "Storage is already tidy"); + showToast(maintenanceOutcome(report).message); void load({ force: true, silent: true }); void loadDiagnostics(); } catch (err) { @@ -778,8 +778,7 @@ export function StorageSection() { confirmLabel: "Clean up safely", runMaintenance: runMaintenanceNow, onMaintenanceDone: (report) => { - const reclaimed = typeof report?.reclaimedBytes === "number" && Number.isFinite(report.reclaimedBytes) ? report.reclaimedBytes : 0; - showToast(reclaimed > 0 ? `Reclaimed ${formatBytes(reclaimed)}` : "Storage is already tidy"); + showToast(maintenanceOutcome(report).message); }, }, }; @@ -820,7 +819,7 @@ export function StorageSection() { snapshot={snapshot} pressureState={pressureState} refreshing={refreshing} - reclaimableBytes={reclaimable} + reclaimableBytes={runMaintenanceNow ? (safeConfig?.plan.estimatedBytes ?? 0) : reclaimable} onRescan={() => void load({ force: true })} onCleanSafely={safeConfig ? () => setSafeOpen(true) : null} /> @@ -923,7 +922,7 @@ export function StorageSection() { setSafeOpen(false)} diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx index 496c0b192..365e0dc88 100644 --- a/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx +++ b/apps/desktop/src/renderer/components/settings/storage/StorageCleanupDialog.tsx @@ -13,7 +13,7 @@ import { dangerButton, primaryButton, } from "../../lanes/laneDesignTokens"; -import { baseName, formatBytes, type SafeCleanupGroup } from "./storageView"; +import { baseName, formatBytes, maintenanceOutcome, type SafeCleanupGroup } from "./storageView"; /** * Optional "safe cleanup" plan. When present the dialog runs the broad @@ -251,6 +251,7 @@ export function StorageCleanupDialog({ (skipPreview ? false : removableCount === 0); const confirmLabel = plan?.confirmLabel ?? (removableCount > 0 ? `Remove ${removableCount === 1 ? "1 item" : `${removableCount} items`}` : "Remove"); + const reportOutcome = report ? maintenanceOutcome(report) : null; return (
-
- {result.freedBytes > 0 - ? `Freed ${formatBytes(result.freedBytes)}.` - : maintenanceMode - ? "Storage is already tidy." +
+ {maintenanceMode && reportOutcome + ? `${reportOutcome.message}.` + : result.freedBytes > 0 + ? `Freed ${formatBytes(result.freedBytes)}.` : "Nothing needed removing."}
- {maintenanceMode && report && report.actions.some((a) => a.error) ? ( -
- Some steps couldn't finish. You can try again later. -
- ) : null} {result.removed.length > 0 ? (
Removed {result.removed.length === 1 ? "1 item" : `${result.removed.length} items`}. diff --git a/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx b/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx index 9ca37304a..4ddadba3c 100644 --- a/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx +++ b/apps/desktop/src/renderer/components/settings/storage/StorageDiagnostics.tsx @@ -130,6 +130,21 @@ export function DiagnosticsStrip({ const lastRun = lastMaintenanceRun(extras); const level = usage ? appResourcePressureLevel(usage) : 0; const health = healthChip(level); + const hasCpuSignal = Boolean(usage && [ + usage.cpuPercent, + usage.mainCpuPercent, + usage.rendererCpuPercent, + usage.ptyCpuPercent, + ].some((value) => typeof value === "number" && Number.isFinite(value))); + const hasMemorySignal = Boolean( + usage + && typeof usage.totalMemoryMB === "number" + && Number.isFinite(usage.totalMemoryMB) + && usage.totalMemoryMB > 0 + && [usage.memoryMB, usage.freeMemoryMB] + .some((value) => typeof value === "number" && Number.isFinite(value)), + ); + const hasPressureSignal = hasCpuSignal || hasMemorySignal; const healthColor = health.tone === "busy" ? COLORS.danger : health.tone === "elevated" ? COLORS.warning : COLORS.success; const notAvailable = Not available yet; @@ -148,7 +163,7 @@ export function DiagnosticsStrip({ How the background service is doing on this Mac.
- {usageReady && usage ? ( + {usageReady && hasPressureSignal ? ( {health.label} diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts index 42562a98d..7c2227308 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts @@ -25,6 +25,7 @@ import { ledgerLabel, maintenanceActionLines, maintenanceHeadline, + maintenanceOutcome, safeReclaimableBytes, sparklinePoints, } from "./storageView"; @@ -132,6 +133,19 @@ describe("maintenanceActionLines", () => { expect(failed.failed).toBe(true); expect(failed.detail).toBe("couldn't finish"); }); + + it("reports partial failures instead of calling a zero-byte run tidy", () => { + const report = run({ + reclaimedBytes: 0, + actions: [ + { ledgerId: "fs.tmp", kind: "delete", itemsAffected: 0, bytesReclaimed: 0, durationMs: 4, error: "busy" }, + ], + }); + expect(maintenanceOutcome(report)).toEqual({ + message: "Some cleanup steps couldn't finish", + failed: true, + }); + }); }); describe("formatRunClock / maintenanceHeadline", () => { @@ -254,9 +268,28 @@ describe("buildSafeCleanupPlan", () => { { id: "c1", label: "npm", path: "/proj/.ade/cache/npm", bytes: 300 * MB, fileCount: 5, lastModifiedAt: null, safety: "safe_to_remove" }, ], }, + { + id: "build_release", + bytes: 200 * MB, + fileCount: 2, + safety: "safe_to_remove", + items: [ + { id: "b1", label: "iOS build data", path: "/proj/.ade/cache/ios-simulator/DerivedData", bytes: 200 * MB, fileCount: 2, lastModifiedAt: null, safety: "safe_to_remove" }, + ], + }, + { + id: "recovery_backups", + bytes: 140 * MB, + fileCount: 2, + safety: "review_first", + items: [ + { id: "r-old", label: "Obsolete recovery backup", path: "/proj/.ade/ade.db.recovery-old.bak", bytes: 60 * MB, fileCount: 1, lastModifiedAt: null, safety: "safe_to_remove" }, + { id: "r-new", label: "Newest recovery backup", path: "/proj/.ade/ade.db.recovery-new.bak", bytes: 80 * MB, fileCount: 1, lastModifiedAt: null, safety: "review_first" }, + ], + }, ]; const extras = { - safeReclaimableBytes: 460 * MB, + safeReclaimableBytes: 660 * MB, dbBreakdown: [ { table: "automation_ingress_events", label: "Webhook history", bytes: 40 * MB, category: "webhooks", action: "prunable" }, { table: "core", label: "Core data", bytes: 8 * MB, category: "core", action: null }, @@ -265,15 +298,21 @@ describe("buildSafeCleanupPlan", () => { it("groups compressible history, database rows, and filesystem targets", () => { const plan = buildSafeCleanupPlan({ categories, extras }, new Map()); - expect(plan.estimatedBytes).toBe(460 * MB); + expect(plan.estimatedBytes).toBe(420 * MB); expect(plan.groups.map((g) => g.heading)).toEqual([ "Chats & terminal history", "Project database", "Temporary & rebuildable files", + "Obsolete recovery backups", ]); - expect(plan.fsTargets).toHaveLength(1); - expect(plan.fsBytes).toBe(300 * MB); - expect(plan.fsGroup?.rows).toHaveLength(1); + expect(plan.fsTargets).toHaveLength(2); + expect(plan.fsBytes).toBe(500 * MB); + expect(plan.fsGroup?.rows).toHaveLength(2); + expect(plan.groups.find((group) => group.heading === "Temporary & rebuildable files")?.rows) + .toEqual([{ label: "iOS build data", size: "200 MB" }]); + expect(plan.groups.find((group) => group.heading === "Obsolete recovery backups")?.rows) + .toEqual([{ label: "Obsolete recovery backup", size: "60.0 MB" }]); + expect(plan.groups.flatMap((group) => group.rows).some((row) => row.label === "Newest recovery backup")).toBe(false); expect(plan.whatHappens.at(-1)).toContain("never touched"); expect(plan.whatHappens.at(-1)).toContain("newest backup is always kept"); }); diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.ts index 54255f9a7..6c6f4e1ad 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.ts @@ -331,6 +331,26 @@ export function maintenanceActionLines( })); } +/** Honest one-line outcome for a maintenance report, including partial failures. */ +export function maintenanceOutcome(report: MaintenanceRunReport): { message: string; failed: boolean } { + const reclaimed = typeof report.reclaimedBytes === "number" && Number.isFinite(report.reclaimedBytes) + ? Math.max(0, report.reclaimedBytes) + : 0; + const failed = report.actions.some((action) => Boolean(action.error)); + if (failed) { + return { + message: reclaimed > 0 + ? `Reclaimed ${formatBytes(reclaimed)}, but some cleanup steps couldn't finish` + : "Some cleanup steps couldn't finish", + failed: true, + }; + } + return { + message: reclaimed > 0 ? `Reclaimed ${formatBytes(reclaimed)}` : "Storage is already tidy", + failed: false, + }; +} + function maintenanceActionDetail(action: MaintenanceAction): string { if (action.error) return "couldn't finish"; if (action.skippedReason) return "nothing to do"; @@ -529,7 +549,7 @@ export function buildSafeCleanupPlan( laneIdByKey: Map, ): SafeCleanupPlan { const extras = snapshot.extras; - const estimatedBytes = safeReclaimableBytes(extras); + const daemonEstimatedBytes = safeReclaimableBytes(extras); const groups: SafeCleanupGroup[] = []; const whatHappens: string[] = []; @@ -555,27 +575,59 @@ export function buildSafeCleanupPlan( whatHappens.push("Clean up data ADE keeps but no longer needs."); } - // Filesystem-safe targets (caches + build/release staging) for the fallback path. + // Filesystem-safe targets for the legacy fallback. The doctor has a narrower + // filesystem policy, so only targets it actually reaps are shown in its plan. const fsTargets: StorageCleanupTarget[] = []; const fsRows: Array<{ label: string; size: string }> = []; + const maintenanceFsRows: Array<{ label: string; size: string }> = []; let fsBytes = 0; + let maintenanceFsBytes = 0; for (const category of snapshot.categories) { if (category.id !== "caches" && category.id !== "build_release") continue; for (const entry of cleanableEntries(category.id, category, laneIdByKey)) { fsTargets.push(entry.target); fsRows.push({ label: entry.item.label, size: formatBytes(entry.item.bytes) }); fsBytes += entry.item.bytes; + const doctorDeletesTarget = entry.target.kind === "stale_tmp_staging" + || (entry.target.kind === "rebuildable_cache" + && /[\\/]\.ade[\\/]cache[\\/]ios-simulator[\\/]DerivedData[\\/]?$/.test(entry.target.path)); + if (doctorDeletesTarget) { + maintenanceFsRows.push({ label: entry.item.label, size: formatBytes(entry.item.bytes) }); + maintenanceFsBytes += entry.item.bytes; + } } } const fsGroup: SafeCleanupGroup | null = fsRows.length > 0 ? { heading: "Temporary & rebuildable files", rows: fsRows } : null; - if (fsGroup) groups.push(fsGroup); - if (fsGroup || estimatedBytes > 0) { + if (maintenanceFsRows.length > 0) { + groups.push({ heading: "Temporary & rebuildable files", rows: maintenanceFsRows }); whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); } + // The doctor removes only backups already classified as obsolete, and its + // keepLatest policy always spares the newest copy. + const obsoleteBackups = snapshot.categories + .find((category) => category.id === "recovery_backups") + ?.items.filter((item) => item.safety === "safe_to_remove") ?? []; + const obsoleteBackupBytes = obsoleteBackups.reduce((sum, item) => sum + item.bytes, 0); + if (obsoleteBackups.length > 0) { + groups.push({ + heading: "Obsolete recovery backups", + rows: obsoleteBackups.map((item) => ({ label: item.label, size: formatBytes(item.bytes) })), + }); + whatHappens.push("Remove obsolete recovery backups while keeping the newest backup."); + } + whatHappens.push("Your chats, projects, and active lanes are never touched, and your newest backup is always kept."); + // The daemon estimate includes every filesystem target offered by the legacy + // cleanup API. Remove targets the doctor will not touch, then add disclosed + // obsolete backups (which the daemon intentionally excludes from its CTA sum). + const estimatedBytes = Math.max( + 0, + daemonEstimatedBytes - fsBytes + maintenanceFsBytes + obsoleteBackupBytes, + ); + return { fsTargets, fsBytes, groups, fsGroup, whatHappens, estimatedBytes }; } From 64ae24be16094d79d83664f959072bdf9c4c4d9d Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:42:34 -0400 Subject: [PATCH 09/10] =?UTF-8?q?ship:=20iteration=204=20=E2=80=94=20addre?= =?UTF-8?q?ss=20four=20review=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ade-cli/src/bootstrap.ts | 13 +++++----- .../services/sync/syncPairingStore.test.ts | 24 +++++++++++++++++++ .../src/services/sync/syncPairingStore.ts | 12 ++++++++++ apps/ade-cli/src/services/sync/syncService.ts | 4 ++++ apps/desktop/src/main/main.ts | 12 ++++++---- .../storage/storageInsightsService.test.ts | 12 ++++++++++ .../storage/storageMaintenanceJournal.ts | 6 +++-- .../settings/StorageSection.test.tsx | 11 +++++---- .../settings/storage/storageView.test.ts | 5 ++-- .../settings/storage/storageView.ts | 20 ++++++++++++---- 10 files changed, 96 insertions(+), 23 deletions(-) diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index 96d7c34f2..b791f7a52 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -504,14 +504,14 @@ export async function createAdeRuntime(args: { const diskPressureMonitor = createDiskPressureMonitor({ roots: [projectRoot, resolveMachineAdeLayout().adeDir], }); - // A sync-enabled runtime must prove zero connected peers before CRR - // compaction. Runtimes without sync have no peer transport and can safely - // report zero immediately. - let liveSyncPeerCount: number | null = resolvedArgs.syncRuntime?.enabled ? null : 0; + // A sync-enabled runtime must prove its durable pairing registry is empty + // before CRR compaction. Offline paired devices still require dedupe history. + // Runtimes without sync have no peer transport and can report zero now. + let registeredSyncPeerCount: number | null = resolvedArgs.syncRuntime?.enabled ? null : 0; let db: AdeDb; try { db = await openKvDb(paths.dbPath, logger, { - hasSyncPeers: () => liveSyncPeerCount !== 0, + hasSyncPeers: () => registeredSyncPeerCount !== 0, }); } catch (error) { const code = mapKvDbOpenErrorCode(classifySqliteOpenError(error)); @@ -1646,10 +1646,11 @@ export async function createAdeRuntime(args: { cloudRelayStore, syncTunnelClientService, onStatusChanged: (snapshot) => { - liveSyncPeerCount = snapshot.connectedPeers.length; + registeredSyncPeerCount = syncService?.getRegisteredPeerCount() ?? null; pushEvent("runtime", { type: "sync-status", snapshot }); }, }); + registeredSyncPeerCount = syncService.getRegisteredPeerCount(); syncServiceForPtyEvents = syncService; } diff --git a/apps/ade-cli/src/services/sync/syncPairingStore.test.ts b/apps/ade-cli/src/services/sync/syncPairingStore.test.ts index f0ea2af87..54a17e829 100644 --- a/apps/ade-cli/src/services/sync/syncPairingStore.test.ts +++ b/apps/ade-cli/src/services/sync/syncPairingStore.test.ts @@ -64,6 +64,30 @@ describe("sync SSH pairing trust", () => { expect(fs.readFileSync(filePath, "utf8")).not.toContain(paired.secret); }); + it("counts durable pairings independently of live connections", () => { + const { store } = createStore(); + const peer = { + deviceId: "offline-phone-1", + deviceName: "Offline iPhone", + platform: "iOS", + deviceType: "phone", + siteId: "offline-ios-site-1", + dbVersion: 0, + } satisfies SyncPeerMetadata; + + expect(store.countPairingRecords()).toBe(0); + store.pairPeerViaLocalTrust(peer); + expect(store.countPairingRecords()).toBe(1); + store.revoke(peer.deviceId); + expect(store.countPairingRecords()).toBe(0); + }); + + it("does not treat an unreadable pairing registry as empty", () => { + const { filePath, store } = createStore(); + fs.writeFileSync(filePath, "not json"); + expect(store.countPairingRecords()).toBeNull(); + }); + it("grants desktop runtime access only after the SSH trust gate", () => { const { store } = createStore(); const peer = { diff --git a/apps/ade-cli/src/services/sync/syncPairingStore.ts b/apps/ade-cli/src/services/sync/syncPairingStore.ts index edc6b811f..3b4905009 100644 --- a/apps/ade-cli/src/services/sync/syncPairingStore.ts +++ b/apps/ade-cli/src/services/sync/syncPairingStore.ts @@ -311,6 +311,18 @@ export function createSyncPairingStore(args: SyncPairingStoreArgs) { return readRecords()[normalized] != null; }, + countPairingRecords(): number | null { + if (!fs.existsSync(args.filePath)) return 0; + try { + const parsed: unknown = JSON.parse(fs.readFileSync(args.filePath, "utf8")); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + return Object.keys(parsed).length; + } catch { + // A damaged registry is not evidence that there are no paired peers. + return null; + } + }, + revoke(deviceId: string): void { const normalized = deviceId.trim(); if (!normalized) return; diff --git a/apps/ade-cli/src/services/sync/syncService.ts b/apps/ade-cli/src/services/sync/syncService.ts index 132c7f56f..b40ab40ff 100644 --- a/apps/ade-cli/src/services/sync/syncService.ts +++ b/apps/ade-cli/src/services/sync/syncService.ts @@ -1734,6 +1734,10 @@ export function createSyncService(args: SyncServiceArgs) { return deviceRegistryService; }, + getRegisteredPeerCount(): number | null { + return pairingStore.countPairingRecords(); + }, + async dispose(): Promise { disposed = true; syncPeerService.disconnect(); diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index c7f5fb826..b5cd6bde0 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -2171,13 +2171,14 @@ app.whenReady().then(async () => { } }; - // Fail closed until sync publishes its first status, then keep the DB - // compaction gate synchronized with the live connected-peer count. - let liveSyncPeerCount: number | null = null; + // Fail closed until the durable pairing registry is available. Offline + // paired devices still need their CRR history, so live connections alone + // are never sufficient evidence that compaction is safe. + let registeredSyncPeerCount: number | null = null; let syncServiceRef: ReturnType | null = null; const db = await measureProjectInitStep("db_open", () => openKvDb(adePaths.dbPath, logger, { - hasSyncPeers: () => liveSyncPeerCount !== 0, + hasSyncPeers: () => registeredSyncPeerCount !== 0, }), ); const keybindingsService = createKeybindingsService({ db }); @@ -3661,7 +3662,7 @@ app.whenReady().then(async () => { projectScaffoldService.listMyGitHubRepos(input), }, onStatusChanged: (snapshot) => { - liveSyncPeerCount = snapshot.connectedPeers.length; + registeredSyncPeerCount = syncServiceRef?.getRegisteredPeerCount() ?? null; const normalizedProjectRoot = normalizeProjectRoot(projectRoot); if (mobileSyncSelectedRoot == null && snapshot.connectedPeers.length > 0) { mobileSyncSelectedRoot = normalizedProjectRoot; @@ -3699,6 +3700,7 @@ app.whenReady().then(async () => { }, }); syncServiceRef = syncService; + registeredSyncPeerCount = syncService.getRegisteredPeerCount(); scheduleBackgroundProjectTask( "sync.initialize", () => measureProjectInitStep("sync.initialize", () => syncService.initialize()), diff --git a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts index 868ca7fd3..e28ed7da2 100644 --- a/apps/desktop/src/main/services/storage/storageInsightsService.test.ts +++ b/apps/desktop/src/main/services/storage/storageInsightsService.test.ts @@ -801,6 +801,18 @@ describe("storageMaintenanceJournal", () => { expect(isMaintenanceReport({ ...validReport, trigger: "sometimes" })).toBe(false); expect(isMaintenanceReport({ ...validReport, actions: [null] })).toBe(false); expect(isMaintenanceReport({ ...validReport, actions: [{ ...validReport.actions[0], durationMs: -1 }] })).toBe(false); + const { skippedReason: _skippedReason, ...withoutSkippedReason } = validReport.actions[0]!; + const { error: _error, ...withoutError } = validReport.actions[0]!; + expect(isMaintenanceReport({ ...validReport, actions: [withoutSkippedReason] })).toBe(false); + expect(isMaintenanceReport({ ...validReport, actions: [withoutError] })).toBe(false); + expect(isMaintenanceReport({ + ...validReport, + actions: [{ ...validReport.actions[0], skippedReason: undefined }], + })).toBe(false); + expect(isMaintenanceReport({ + ...validReport, + actions: [{ ...validReport.actions[0], error: undefined }], + })).toBe(false); expect(isMaintenanceReport({ ...validReport, dbSizeBytes: undefined })).toBe(false); }); diff --git a/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts index 699edfbf0..c5c417b4d 100644 --- a/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts +++ b/apps/desktop/src/main/services/storage/storageMaintenanceJournal.ts @@ -17,8 +17,8 @@ function isFiniteNonNegativeNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value >= 0; } -function isNullableString(value: unknown): value is string | null | undefined { - return value == null || typeof value === "string"; +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === "string"; } function isMaintenanceAction(value: unknown): boolean { @@ -31,7 +31,9 @@ function isMaintenanceAction(value: unknown): boolean { && isFiniteNonNegativeNumber(action.itemsAffected) && isFiniteNonNegativeNumber(action.bytesReclaimed) && isFiniteNonNegativeNumber(action.durationMs) + && Object.hasOwn(action, "skippedReason") && isNullableString(action.skippedReason) + && Object.hasOwn(action, "error") && isNullableString(action.error); } diff --git a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx index 3481f65d3..99ade5b3d 100644 --- a/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx +++ b/apps/desktop/src/renderer/components/settings/StorageSection.test.tsx @@ -188,6 +188,7 @@ function makeUsage(): AppResourceUsageSnapshot { function installAdeMock(options: { withCompress?: boolean; withExtras?: boolean; + extras?: StorageSnapshotExtras; withApp?: boolean; maintenanceReport?: MaintenanceRunReport; resourceUsage?: AppResourceUsageSnapshot; @@ -226,15 +227,17 @@ function installAdeMock(options: { const getResourceUsage = vi.fn(async () => options.resourceUsage ?? makeUsage()); const storage: Record = { - getSnapshot: vi.fn(async () => (options.withExtras ? makeSnapshotWithExtras() : makeSnapshot())), + getSnapshot: vi.fn(async () => options.extras + ? { ...makeSnapshot(), extras: options.extras } + : options.withExtras ? makeSnapshotWithExtras() : makeSnapshot()), getPressure: vi.fn(async () => ({ state: "normal", freeBytes: 40 * 1024 ** 3, totalBytes: 500 * 1024 ** 3, freeFraction: 0.08, perRoot: [], sampledAt: new Date().toISOString() })), cleanupPreview, cleanup: cleanupFn, }; if (options.withCompress) storage.compressNow = compressNow; - if (options.withExtras) storage.runMaintenanceNow = runMaintenanceNow; + if (options.withExtras || options.extras) storage.runMaintenanceNow = runMaintenanceNow; - const includeApp = options.withApp ?? options.withExtras; + const includeApp = options.withApp ?? (options.withExtras || options.extras != null); (globalThis.window as any).ade = { storage, lanes: { @@ -350,7 +353,7 @@ describe("StorageSection", () => { }); it("hides the safe-cleanup primary when the daemon reports no reclaimable space", async () => { - installAdeMock(); + installAdeMock({ extras: { ...makeExtras(), safeReclaimableBytes: 0 } }); render(); await screen.findByText(/ADE is using/); expect(screen.queryByRole("button", { name: /clean up safely/i })).toBeNull(); diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts index 7c2227308..34c3d32ba 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.test.ts @@ -283,8 +283,8 @@ describe("buildSafeCleanupPlan", () => { fileCount: 2, safety: "review_first", items: [ - { id: "r-old", label: "Obsolete recovery backup", path: "/proj/.ade/ade.db.recovery-old.bak", bytes: 60 * MB, fileCount: 1, lastModifiedAt: null, safety: "safe_to_remove" }, - { id: "r-new", label: "Newest recovery backup", path: "/proj/.ade/ade.db.recovery-new.bak", bytes: 80 * MB, fileCount: 1, lastModifiedAt: null, safety: "review_first" }, + { id: "r-old", label: "Obsolete recovery backup", path: "/proj/.ade/ade.db.recovery-old.bak", bytes: 60 * MB, fileCount: 1, lastModifiedAt: "2026-06-01T00:00:00.000Z", safety: "safe_to_remove" }, + { id: "r-new", label: "Newest recovery backup", path: "/proj/.ade/ade.db.recovery-new.bak", bytes: 80 * MB, fileCount: 1, lastModifiedAt: "2026-07-01T00:00:00.000Z", safety: "safe_to_remove" }, ], }, ]; @@ -313,6 +313,7 @@ describe("buildSafeCleanupPlan", () => { expect(plan.groups.find((group) => group.heading === "Obsolete recovery backups")?.rows) .toEqual([{ label: "Obsolete recovery backup", size: "60.0 MB" }]); expect(plan.groups.flatMap((group) => group.rows).some((row) => row.label === "Newest recovery backup")).toBe(false); + expect(plan.estimatedBytes).not.toBe(500 * MB); expect(plan.whatHappens.at(-1)).toContain("never touched"); expect(plan.whatHappens.at(-1)).toContain("newest backup is always kept"); }); diff --git a/apps/desktop/src/renderer/components/settings/storage/storageView.ts b/apps/desktop/src/renderer/components/settings/storage/storageView.ts index 6c6f4e1ad..800b423ef 100644 --- a/apps/desktop/src/renderer/components/settings/storage/storageView.ts +++ b/apps/desktop/src/renderer/components/settings/storage/storageView.ts @@ -605,11 +605,23 @@ export function buildSafeCleanupPlan( whatHappens.push("Remove temporary and rebuildable files ADE recreates on demand."); } - // The doctor removes only backups already classified as obsolete, and its - // keepLatest policy always spares the newest copy. - const obsoleteBackups = snapshot.categories + // Independently enforce the doctor's keepLatest: 1 policy instead of + // trusting the snapshot's safety classification. A stale snapshot may still + // label the newest backup safe-to-remove; the doctor will spare it, so the + // plan must not show or estimate it. Snapshot order is the deterministic + // fallback when modification times are unavailable. + const recoveryBackups = snapshot.categories .find((category) => category.id === "recovery_backups") - ?.items.filter((item) => item.safety === "safe_to_remove") ?? []; + ?.items ?? []; + const newestBackup = recoveryBackups.reduce((newest, item) => { + if (!newest) return item; + const newestModifiedAt = newest.lastModifiedAt ? Date.parse(newest.lastModifiedAt) : Number.NEGATIVE_INFINITY; + const itemModifiedAt = item.lastModifiedAt ? Date.parse(item.lastModifiedAt) : Number.NEGATIVE_INFINITY; + return itemModifiedAt > newestModifiedAt ? item : newest; + }, null); + const obsoleteBackups = recoveryBackups.filter( + (item) => item !== newestBackup && item.safety === "safe_to_remove", + ); const obsoleteBackupBytes = obsoleteBackups.reduce((sum, item) => sum + item.bytes, 0); if (obsoleteBackups.length > 0) { groups.push({ From 88d5def4ae7c367b6d5303b14add7e3ce100ff90 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Sat, 18 Jul 2026 01:08:26 -0400 Subject: [PATCH 10/10] =?UTF-8?q?ship:=20iteration=205=20=E2=80=94=20refre?= =?UTF-8?q?sh=20peer=20compaction=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ade-cli/src/bootstrap.ts | 18 +++---- apps/desktop/src/main/main.ts | 13 +++-- .../state/syncPeerCompactionGate.test.ts | 48 +++++++++++++++++++ .../services/state/syncPeerCompactionGate.ts | 21 ++++++++ 4 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 apps/desktop/src/main/services/state/syncPeerCompactionGate.test.ts create mode 100644 apps/desktop/src/main/services/state/syncPeerCompactionGate.ts diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index b791f7a52..663fd6549 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -7,6 +7,7 @@ import * as nodePty from "node-pty"; import { isSourceCheckoutRuntimeModule } from "./runtimePackaging"; import { createFileLogger, type Logger } from "../../desktop/src/main/services/logging/logger"; import { classifySqliteOpenError, openKvDb, type AdeDb } from "../../desktop/src/main/services/state/kvDb"; +import { createRegisteredSyncPeerGate } from "../../desktop/src/main/services/state/syncPeerCompactionGate"; import { clearLastFailure, recordLastFailure, @@ -504,14 +505,15 @@ export async function createAdeRuntime(args: { const diskPressureMonitor = createDiskPressureMonitor({ roots: [projectRoot, resolveMachineAdeLayout().adeDir], }); - // A sync-enabled runtime must prove its durable pairing registry is empty - // before CRR compaction. Offline paired devices still require dedupe history. - // Runtimes without sync have no peer transport and can report zero now. - let registeredSyncPeerCount: number | null = resolvedArgs.syncRuntime?.enabled ? null : 0; + let syncService: ReturnType | null = null; + const hasSyncPeers = createRegisteredSyncPeerGate({ + syncEnabled: resolvedArgs.syncRuntime?.enabled === true, + getSyncService: () => syncService, + }); let db: AdeDb; try { db = await openKvDb(paths.dbPath, logger, { - hasSyncPeers: () => registeredSyncPeerCount !== 0, + hasSyncPeers, }); } catch (error) { const code = mapKvDbOpenErrorCode(classifySqliteOpenError(error)); @@ -1586,7 +1588,6 @@ export async function createAdeRuntime(args: { } let externalSessionsService: ReturnType | null = null; - let syncService: ReturnType | null = null; if (resolvedArgs.syncRuntime?.enabled && agentChatService) { const { createSyncService } = await import("./services/sync/syncService"); syncService = createSyncService({ @@ -1646,18 +1647,17 @@ export async function createAdeRuntime(args: { cloudRelayStore, syncTunnelClientService, onStatusChanged: (snapshot) => { - registeredSyncPeerCount = syncService?.getRegisteredPeerCount() ?? null; pushEvent("runtime", { type: "sync-status", snapshot }); }, }); - registeredSyncPeerCount = syncService.getRegisteredPeerCount(); syncServiceForPtyEvents = syncService; } if (syncService) { + const currentSyncService = syncService; const initializeSyncService = async () => { try { - await syncService.initialize(); + await currentSyncService.initialize(); } catch (error) { logger.warn("sync.runtime_initialize_failed", { error: error instanceof Error ? error.message : String(error), diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index b5cd6bde0..f1df01ee3 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -32,6 +32,7 @@ import { initPerfRunFromEnv } from "./services/perf/perfLog"; import { startMetricsSampler } from "./services/perf/metricsSampler"; import { registerPerfIpcHandlers } from "./services/perf/perfIpc"; import { openKvDb } from "./services/state/kvDb"; +import { createRegisteredSyncPeerGate } from "./services/state/syncPeerCompactionGate"; import { ensureAdeDirs } from "./services/state/projectState"; import { persistableRemoteProjectIconDataUrl, @@ -2171,14 +2172,14 @@ app.whenReady().then(async () => { } }; - // Fail closed until the durable pairing registry is available. Offline - // paired devices still need their CRR history, so live connections alone - // are never sufficient evidence that compaction is safe. - let registeredSyncPeerCount: number | null = null; let syncServiceRef: ReturnType | null = null; + const hasSyncPeers = createRegisteredSyncPeerGate({ + syncEnabled: true, + getSyncService: () => syncServiceRef, + }); const db = await measureProjectInitStep("db_open", () => openKvDb(adePaths.dbPath, logger, { - hasSyncPeers: () => registeredSyncPeerCount !== 0, + hasSyncPeers, }), ); const keybindingsService = createKeybindingsService({ db }); @@ -3662,7 +3663,6 @@ app.whenReady().then(async () => { projectScaffoldService.listMyGitHubRepos(input), }, onStatusChanged: (snapshot) => { - registeredSyncPeerCount = syncServiceRef?.getRegisteredPeerCount() ?? null; const normalizedProjectRoot = normalizeProjectRoot(projectRoot); if (mobileSyncSelectedRoot == null && snapshot.connectedPeers.length > 0) { mobileSyncSelectedRoot = normalizedProjectRoot; @@ -3700,7 +3700,6 @@ app.whenReady().then(async () => { }, }); syncServiceRef = syncService; - registeredSyncPeerCount = syncService.getRegisteredPeerCount(); scheduleBackgroundProjectTask( "sync.initialize", () => measureProjectInitStep("sync.initialize", () => syncService.initialize()), diff --git a/apps/desktop/src/main/services/state/syncPeerCompactionGate.test.ts b/apps/desktop/src/main/services/state/syncPeerCompactionGate.test.ts new file mode 100644 index 000000000..1376c47c0 --- /dev/null +++ b/apps/desktop/src/main/services/state/syncPeerCompactionGate.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { createRegisteredSyncPeerGate, type RegisteredSyncPeerCounter } from "./syncPeerCompactionGate"; + +describe("createRegisteredSyncPeerGate", () => { + it("reads the durable peer count at every compaction decision", () => { + let registeredPeerCount: number | null = 0; + const syncService: RegisteredSyncPeerCounter = { + getRegisteredPeerCount: () => registeredPeerCount, + }; + const hasSyncPeers = createRegisteredSyncPeerGate({ + syncEnabled: true, + getSyncService: () => syncService, + }); + + expect(hasSyncPeers()).toBe(false); + registeredPeerCount = 1; + expect(hasSyncPeers()).toBe(true); + registeredPeerCount = 0; + expect(hasSyncPeers()).toBe(false); + }); + + it("fails closed before the service exists and when the registry is unreadable", () => { + let syncService: RegisteredSyncPeerCounter | null = null; + const hasSyncPeers = createRegisteredSyncPeerGate({ + syncEnabled: true, + getSyncService: () => syncService, + }); + + expect(hasSyncPeers()).toBe(true); + syncService = { getRegisteredPeerCount: () => null }; + expect(hasSyncPeers()).toBe(true); + syncService = { + getRegisteredPeerCount: () => { + throw new Error("registry unreadable"); + }, + }; + expect(hasSyncPeers()).toBe(true); + }); + + it("allows compaction when sync is disabled", () => { + const hasSyncPeers = createRegisteredSyncPeerGate({ + syncEnabled: false, + getSyncService: () => null, + }); + + expect(hasSyncPeers()).toBe(false); + }); +}); diff --git a/apps/desktop/src/main/services/state/syncPeerCompactionGate.ts b/apps/desktop/src/main/services/state/syncPeerCompactionGate.ts new file mode 100644 index 000000000..fcfaca17c --- /dev/null +++ b/apps/desktop/src/main/services/state/syncPeerCompactionGate.ts @@ -0,0 +1,21 @@ +export type RegisteredSyncPeerCounter = { + getRegisteredPeerCount(): number | null; +}; + +export function createRegisteredSyncPeerGate(args: { + syncEnabled: boolean; + getSyncService: () => RegisteredSyncPeerCounter | null; +}): () => boolean { + return () => { + if (!args.syncEnabled) return false; + + try { + const syncService = args.getSyncService(); + return syncService == null || syncService.getRegisteredPeerCount() !== 0; + } catch { + // Missing or unreadable durable pairing state is not evidence that + // compaction is safe. Keep CRR history until the registry is readable. + return true; + } + }; +}