From 31843c52907f7263de0313c89c85b45fc21fbf2f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 13 Mar 2026 10:03:13 -0700 Subject: [PATCH 01/19] Add preliminary indexing dashboard --- .../handlers/handle-indexing-dashboard.ts | 625 ++++++++++++++++++ packages/realm-server/routes.ts | 5 + 2 files changed, 630 insertions(+) create mode 100644 packages/realm-server/handlers/handle-indexing-dashboard.ts diff --git a/packages/realm-server/handlers/handle-indexing-dashboard.ts b/packages/realm-server/handlers/handle-indexing-dashboard.ts new file mode 100644 index 0000000000..cede33433d --- /dev/null +++ b/packages/realm-server/handlers/handle-indexing-dashboard.ts @@ -0,0 +1,625 @@ +import type Koa from 'koa'; +import { query } from '@cardstack/runtime-common'; +import { setContextResponse } from '../middleware'; +import type { CreateRoutesArgs } from '../routes'; + +interface IndexingJob { + id: number; + job_type: string; + concurrency_group: string | null; + status: string; + priority: number; + args: Record | null; + created_at: string; + finished_at: string | null; + result: Record | null; + worker_id: string | null; + reservation_started: string | null; + locked_until: string | null; +} + +interface RealmIndexInfo { + realm_url: string; + total_entries: number; + instances: number; + files: number; + errors: number; +} + +interface WorkingEntry { + realm_url: string; + working_count: number; +} + +async function getIndexingData(dbAdapter: CreateRoutesArgs['dbAdapter']) { + let [activeJobs, recentJobs, realmIndex, workingEntries] = await Promise.all([ + // Active/pending indexing jobs with reservation info + query(dbAdapter, [ + `SELECT + j.id, j.job_type, j.concurrency_group, j.status, j.priority, + j.args, j.created_at, j.finished_at, j.result, + jr.worker_id, jr.created_at as reservation_started, jr.locked_until + FROM jobs j + LEFT JOIN job_reservations jr ON jr.job_id = j.id AND jr.completed_at IS NULL + WHERE j.job_type IN ('from-scratch-index', 'incremental-index', 'copy-index') + AND j.status = 'unfulfilled' + ORDER BY j.priority DESC, j.created_at`, + ]) as unknown as IndexingJob[], + + // Recent completed jobs + query(dbAdapter, [ + `SELECT + j.id, j.job_type, j.concurrency_group, j.status, j.priority, + j.args, j.created_at, j.finished_at, j.result, + NULL as worker_id, NULL as reservation_started, NULL as locked_until + FROM jobs j + WHERE j.job_type IN ('from-scratch-index', 'incremental-index', 'copy-index') + AND j.status IN ('resolved', 'rejected') + ORDER BY j.finished_at DESC + LIMIT 50`, + ]) as unknown as IndexingJob[], + + // Index entry counts per realm + query(dbAdapter, [ + `SELECT + realm_url, + CAST(COUNT(*) AS INTEGER) as total_entries, + CAST(COUNT(*) FILTER (WHERE type = 'instance') AS INTEGER) as instances, + CAST(COUNT(*) FILTER (WHERE type = 'file') AS INTEGER) as files, + CAST(COUNT(*) FILTER (WHERE has_error = true) AS INTEGER) as errors + FROM boxel_index + WHERE is_deleted IS NOT TRUE + GROUP BY realm_url + ORDER BY realm_url`, + ]) as unknown as RealmIndexInfo[], + + // Working entries per realm (shows in-progress batch work) + query(dbAdapter, [ + `SELECT + realm_url, + CAST(COUNT(*) AS INTEGER) as working_count + FROM boxel_index_working + GROUP BY realm_url`, + ]) as unknown as WorkingEntry[], + ]); + + return { activeJobs, recentJobs, realmIndex, workingEntries }; +} + +function extractRealmURL(job: IndexingJob): string { + if (job.args && typeof job.args === 'object' && 'realmURL' in job.args) { + return job.args.realmURL as string; + } + if (job.concurrency_group) { + return job.concurrency_group.replace('indexing:', ''); + } + return 'unknown'; +} + +function extractChanges(job: IndexingJob): string[] { + if ( + job.args && + typeof job.args === 'object' && + 'changes' in job.args && + Array.isArray(job.args.changes) + ) { + return job.args.changes.map( + (c: { url: string; operation: string }) => + `${c.operation}: ${c.url}`, + ); + } + return []; +} + +function extractStats( + job: IndexingJob, +): Record | null { + if ( + job.result && + typeof job.result === 'object' && + 'stats' in job.result + ) { + return job.result.stats as Record; + } + return null; +} + +function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function timeSince(dateStr: string): string { + let date = new Date(dateStr); + let seconds = Math.floor((Date.now() - date.getTime()) / 1000); + if (seconds < 60) { + return `${seconds}s ago`; + } + let minutes = Math.floor(seconds / 60); + if (minutes < 60) { + return `${minutes}m ago`; + } + let hours = Math.floor(minutes / 60); + if (hours < 24) { + return `${hours}h ${minutes % 60}m ago`; + } + let days = Math.floor(hours / 24); + return `${days}d ago`; +} + +function duration(startStr: string, endStr: string | null): string { + if (!endStr) { + return 'in progress'; + } + let ms = new Date(endStr).getTime() - new Date(startStr).getTime(); + if (ms < 1000) { + return `${ms}ms`; + } + let seconds = Math.floor(ms / 1000); + if (seconds < 60) { + return `${seconds}s`; + } + let minutes = Math.floor(seconds / 60); + return `${minutes}m ${seconds % 60}s`; +} + +function renderDashboard(data: Awaited>): string { + let { activeJobs, recentJobs, realmIndex, workingEntries } = data; + + let workingMap = new Map( + workingEntries.map((w) => [w.realm_url, w.working_count]), + ); + + // Group active jobs by realm + let activeByRealm = new Map(); + for (let job of activeJobs) { + let realmURL = extractRealmURL(job); + let jobs = activeByRealm.get(realmURL) || []; + jobs.push(job); + activeByRealm.set(realmURL, jobs); + } + + // Build realm status cards + let realmCards = realmIndex + .map((realm) => { + let active = activeByRealm.get(realm.realm_url) || []; + let working = workingMap.get(realm.realm_url) || 0; + let isIndexing = active.length > 0; + let statusClass = isIndexing ? 'indexing' : 'idle'; + + let activeJobsHtml = ''; + if (active.length > 0) { + activeJobsHtml = active + .map((job) => { + let changes = extractChanges(job); + let jobTypeLabel = job.job_type.replace('-', ' '); + let startedInfo = job.reservation_started + ? `started ${timeSince(job.reservation_started)}` + : `queued ${timeSince(job.created_at)}`; + let workerInfo = job.worker_id + ? ` (worker: ${escapeHtml(job.worker_id.substring(0, 8))})` + : ''; + + let changesHtml = ''; + if (changes.length > 0) { + let changesList = changes + .map((c) => `
  • ${escapeHtml(c)}
    • `) + .join(''); + changesHtml = ` +
    + ${changes.length} file${changes.length !== 1 ? 's' : ''} to process +
      ${changesList}
    +
    `; + } + + return ` +
    +
    + ${escapeHtml(jobTypeLabel)} + ${escapeHtml(startedInfo)}${escapeHtml(workerInfo)} +
    + ${changesHtml} + ${working > 0 ? `
    ${working} entries written to working index
    ` : ''} +
    `; + }) + .join(''); + } + + return ` +
    +
    + +

    ${escapeHtml(realm.realm_url)}

    +
    +
    + ${realm.total_entries} entries + ${realm.instances} instances + ${realm.files} files + ${realm.errors > 0 ? `${realm.errors} errors` : ''} +
    + ${activeJobsHtml} +
    `; + }) + .join(''); + + // Also show realms that have active jobs but no index entries yet + for (let [realmURL, jobs] of activeByRealm) { + if (!realmIndex.find((r) => r.realm_url === realmURL)) { + let working = workingMap.get(realmURL) || 0; + let jobsHtml = jobs + .map((job) => { + let jobTypeLabel = job.job_type.replace('-', ' '); + let startedInfo = job.reservation_started + ? `started ${timeSince(job.reservation_started)}` + : `queued ${timeSince(job.created_at)}`; + let changes = extractChanges(job); + let changesHtml = ''; + if (changes.length > 0) { + let changesList = changes + .map((c) => `
  • ${escapeHtml(c)}
    • `) + .join(''); + changesHtml = ` +
    + ${changes.length} file${changes.length !== 1 ? 's' : ''} to process +
      ${changesList}
    +
    `; + } + return ` +
    +
    + ${escapeHtml(jobTypeLabel)} + ${escapeHtml(startedInfo)} +
    + ${changesHtml} + ${working > 0 ? `
    ${working} entries written to working index
    ` : ''} +
    `; + }) + .join(''); + + realmCards += ` +
    +
    + +

    ${escapeHtml(realmURL)}

    +
    +
    + 0 entries (new index) +
    + ${jobsHtml} +
    `; + } + } + + // Recent completed jobs table + let recentJobsRows = recentJobs + .map((job) => { + let realmURL = extractRealmURL(job); + let stats = extractStats(job); + let statsHtml = stats + ? Object.entries(stats) + .map(([k, v]) => `${k}: ${v}`) + .join(', ') + : ''; + let statusClass = job.status === 'resolved' ? 'success' : 'failure'; + return ` + + ${job.id} + ${escapeHtml(job.job_type.replace('-', ' '))} + ${escapeHtml(realmURL)} + ${escapeHtml(job.status)} + ${duration(job.created_at, job.finished_at)} + ${job.finished_at ? timeSince(job.finished_at) : ''} + ${escapeHtml(statsHtml)} + `; + }) + .join(''); + + return ` + + + + Indexing Dashboard + + + + +
    +

    Indexing Dashboard

    +
    + + +
    +
    + +
    +
    +
    ${activeJobs.length}
    +
    Active Jobs
    +
    +
    +
    ${realmIndex.length}
    +
    Realms
    +
    +
    +
    ${realmIndex.reduce((s, r) => s + r.total_entries, 0)}
    +
    Total Entries
    +
    +
    +
    ${realmIndex.reduce((s, r) => s + r.errors, 0)}
    +
    Total Errors
    +
    +
    + +

    Realms

    + ${realmCards.length > 0 ? `
    ${realmCards}
    ` : '
    No realms found
    '} + +

    Recent Jobs

    + ${ + recentJobs.length > 0 + ? `
    + + + + + + + + + + + + + ${recentJobsRows} +
    IDTypeRealmStatusDurationFinishedStats
    +
    ` + : '
    No recent jobs
    ' + } + + + +`; +} + +export default function handleIndexingDashboard({ + dbAdapter, + grafanaSecret, +}: CreateRoutesArgs): (ctxt: Koa.Context, next: Koa.Next) => Promise { + return async function (ctxt: Koa.Context, _next: Koa.Next) { + let authorization = ctxt.req.headers['authorization']; + let queryToken = ctxt.query['token']; + + // Allow auth via header or query param (for easy browser access) + if (authorization !== grafanaSecret && queryToken !== grafanaSecret) { + return setContextResponse( + ctxt, + new Response('Unauthorized - provide grafana secret as Authorization header or ?token= query param', { + status: 401, + }), + ); + } + + let data = await getIndexingData(dbAdapter); + let html = renderDashboard(data); + + return setContextResponse( + ctxt, + new Response(html, { + headers: { 'content-type': 'text/html; charset=utf-8' }, + }), + ); + }; +} diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index 9fbcceaac4..efcb2d4bd8 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -66,6 +66,7 @@ import { handleDeleteWebhookCommandRequest, } from './handlers/handle-webhook-commands'; import handleWebhookReceiverRequest from './handlers/handle-webhook-receiver'; +import handleIndexingDashboard from './handlers/handle-indexing-dashboard'; import { buildCreatePrerenderAuth } from './prerender/auth'; export type CreateRoutesArgs = { @@ -238,6 +239,10 @@ export function createRoutes(args: CreateRoutesArgs) { grafanaAuthorization(args.grafanaSecret), handleFullReindex(args), ); + router.get( + '/_indexing-dashboard', + handleIndexingDashboard(args), + ); router.post('/_post-deployment', handlePostDeployment(args)); router.post( '/_realm-auth', From ab51a9cd2cdca089817957f4e2a9ab8741f1949c Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 13 Mar 2026 10:13:35 -0700 Subject: [PATCH 02/19] Change to only show on localhost --- .../handlers/handle-indexing-dashboard.ts | 14 -------------- packages/realm-server/routes.ts | 7 +++---- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/packages/realm-server/handlers/handle-indexing-dashboard.ts b/packages/realm-server/handlers/handle-indexing-dashboard.ts index cede33433d..a286703739 100644 --- a/packages/realm-server/handlers/handle-indexing-dashboard.ts +++ b/packages/realm-server/handlers/handle-indexing-dashboard.ts @@ -596,22 +596,8 @@ function renderDashboard(data: Awaited>): str export default function handleIndexingDashboard({ dbAdapter, - grafanaSecret, }: CreateRoutesArgs): (ctxt: Koa.Context, next: Koa.Next) => Promise { return async function (ctxt: Koa.Context, _next: Koa.Next) { - let authorization = ctxt.req.headers['authorization']; - let queryToken = ctxt.query['token']; - - // Allow auth via header or query param (for easy browser access) - if (authorization !== grafanaSecret && queryToken !== grafanaSecret) { - return setContextResponse( - ctxt, - new Response('Unauthorized - provide grafana secret as Authorization header or ?token= query param', { - status: 401, - }), - ); - } - let data = await getIndexingData(dbAdapter); let html = renderDashboard(data); diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index efcb2d4bd8..8ed27f383c 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -239,10 +239,9 @@ export function createRoutes(args: CreateRoutesArgs) { grafanaAuthorization(args.grafanaSecret), handleFullReindex(args), ); - router.get( - '/_indexing-dashboard', - handleIndexingDashboard(args), - ); + if (args.assetsURL.hostname.includes('localhost')) { + router.get('/_indexing-dashboard', handleIndexingDashboard(args)); + } router.post('/_post-deployment', handlePostDeployment(args)); router.post( '/_realm-auth', From ef95dc0806beb5cea737fbfce599b84400598bc1 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 13 Mar 2026 11:30:00 -0700 Subject: [PATCH 03/19] Move dashboard to worker (manager?) --- .../handlers/handle-indexing-dashboard.ts | 495 ++++++------------ packages/realm-server/indexing-event-sink.ts | 92 ++++ packages/realm-server/worker-manager.ts | 23 + packages/realm-server/worker.ts | 8 + packages/runtime-common/index-runner.ts | 53 ++ packages/runtime-common/tasks/index.ts | 3 +- packages/runtime-common/tasks/indexer.ts | 4 + packages/runtime-common/worker.ts | 21 + 8 files changed, 355 insertions(+), 344 deletions(-) create mode 100644 packages/realm-server/indexing-event-sink.ts diff --git a/packages/realm-server/handlers/handle-indexing-dashboard.ts b/packages/realm-server/handlers/handle-indexing-dashboard.ts index a286703739..c7a12a0841 100644 --- a/packages/realm-server/handlers/handle-indexing-dashboard.ts +++ b/packages/realm-server/handlers/handle-indexing-dashboard.ts @@ -1,128 +1,7 @@ import type Koa from 'koa'; -import { query } from '@cardstack/runtime-common'; import { setContextResponse } from '../middleware'; import type { CreateRoutesArgs } from '../routes'; - -interface IndexingJob { - id: number; - job_type: string; - concurrency_group: string | null; - status: string; - priority: number; - args: Record | null; - created_at: string; - finished_at: string | null; - result: Record | null; - worker_id: string | null; - reservation_started: string | null; - locked_until: string | null; -} - -interface RealmIndexInfo { - realm_url: string; - total_entries: number; - instances: number; - files: number; - errors: number; -} - -interface WorkingEntry { - realm_url: string; - working_count: number; -} - -async function getIndexingData(dbAdapter: CreateRoutesArgs['dbAdapter']) { - let [activeJobs, recentJobs, realmIndex, workingEntries] = await Promise.all([ - // Active/pending indexing jobs with reservation info - query(dbAdapter, [ - `SELECT - j.id, j.job_type, j.concurrency_group, j.status, j.priority, - j.args, j.created_at, j.finished_at, j.result, - jr.worker_id, jr.created_at as reservation_started, jr.locked_until - FROM jobs j - LEFT JOIN job_reservations jr ON jr.job_id = j.id AND jr.completed_at IS NULL - WHERE j.job_type IN ('from-scratch-index', 'incremental-index', 'copy-index') - AND j.status = 'unfulfilled' - ORDER BY j.priority DESC, j.created_at`, - ]) as unknown as IndexingJob[], - - // Recent completed jobs - query(dbAdapter, [ - `SELECT - j.id, j.job_type, j.concurrency_group, j.status, j.priority, - j.args, j.created_at, j.finished_at, j.result, - NULL as worker_id, NULL as reservation_started, NULL as locked_until - FROM jobs j - WHERE j.job_type IN ('from-scratch-index', 'incremental-index', 'copy-index') - AND j.status IN ('resolved', 'rejected') - ORDER BY j.finished_at DESC - LIMIT 50`, - ]) as unknown as IndexingJob[], - - // Index entry counts per realm - query(dbAdapter, [ - `SELECT - realm_url, - CAST(COUNT(*) AS INTEGER) as total_entries, - CAST(COUNT(*) FILTER (WHERE type = 'instance') AS INTEGER) as instances, - CAST(COUNT(*) FILTER (WHERE type = 'file') AS INTEGER) as files, - CAST(COUNT(*) FILTER (WHERE has_error = true) AS INTEGER) as errors - FROM boxel_index - WHERE is_deleted IS NOT TRUE - GROUP BY realm_url - ORDER BY realm_url`, - ]) as unknown as RealmIndexInfo[], - - // Working entries per realm (shows in-progress batch work) - query(dbAdapter, [ - `SELECT - realm_url, - CAST(COUNT(*) AS INTEGER) as working_count - FROM boxel_index_working - GROUP BY realm_url`, - ]) as unknown as WorkingEntry[], - ]); - - return { activeJobs, recentJobs, realmIndex, workingEntries }; -} - -function extractRealmURL(job: IndexingJob): string { - if (job.args && typeof job.args === 'object' && 'realmURL' in job.args) { - return job.args.realmURL as string; - } - if (job.concurrency_group) { - return job.concurrency_group.replace('indexing:', ''); - } - return 'unknown'; -} - -function extractChanges(job: IndexingJob): string[] { - if ( - job.args && - typeof job.args === 'object' && - 'changes' in job.args && - Array.isArray(job.args.changes) - ) { - return job.args.changes.map( - (c: { url: string; operation: string }) => - `${c.operation}: ${c.url}`, - ); - } - return []; -} - -function extractStats( - job: IndexingJob, -): Record | null { - if ( - job.result && - typeof job.result === 'object' && - 'stats' in job.result - ) { - return job.result.stats as Record; - } - return null; -} +import type { RealmIndexingState } from '../indexing-event-sink'; function escapeHtml(str: string): string { return str @@ -132,29 +11,21 @@ function escapeHtml(str: string): string { .replace(/"/g, '"'); } -function timeSince(dateStr: string): string { - let date = new Date(dateStr); - let seconds = Math.floor((Date.now() - date.getTime()) / 1000); +function timeSince(ms: number): string { + let seconds = Math.floor((Date.now() - ms) / 1000); if (seconds < 60) { return `${seconds}s ago`; } let minutes = Math.floor(seconds / 60); if (minutes < 60) { - return `${minutes}m ago`; + return `${minutes}m ${seconds % 60}s ago`; } let hours = Math.floor(minutes / 60); - if (hours < 24) { - return `${hours}h ${minutes % 60}m ago`; - } - let days = Math.floor(hours / 24); - return `${days}d ago`; + return `${hours}h ${minutes % 60}m ago`; } -function duration(startStr: string, endStr: string | null): string { - if (!endStr) { - return 'in progress'; - } - let ms = new Date(endStr).getTime() - new Date(startStr).getTime(); +function durationMs(startMs: number, endMs?: number): string { + let ms = (endMs ?? Date.now()) - startMs; if (ms < 1000) { return `${ms}ms`; } @@ -166,156 +37,86 @@ function duration(startStr: string, endStr: string | null): string { return `${minutes}m ${seconds % 60}s`; } -function renderDashboard(data: Awaited>): string { - let { activeJobs, recentJobs, realmIndex, workingEntries } = data; +function renderActiveCard(state: RealmIndexingState): string { + let remaining = state.totalFiles - state.filesCompleted; + let pct = + state.totalFiles > 0 + ? Math.round((state.filesCompleted / state.totalFiles) * 100) + : 0; - let workingMap = new Map( - workingEntries.map((w) => [w.realm_url, w.working_count]), + let remainingFiles = state.files.filter( + (f) => !state.completedFiles.includes(f), ); + let remainingList = remainingFiles + .map((f) => `
  • ${escapeHtml(f)}
    • `) + .join(''); + let completedList = state.completedFiles + .map((f) => `
  • ${escapeHtml(f)}
    • `) + .join(''); - // Group active jobs by realm - let activeByRealm = new Map(); - for (let job of activeJobs) { - let realmURL = extractRealmURL(job); - let jobs = activeByRealm.get(realmURL) || []; - jobs.push(job); - activeByRealm.set(realmURL, jobs); - } - - // Build realm status cards - let realmCards = realmIndex - .map((realm) => { - let active = activeByRealm.get(realm.realm_url) || []; - let working = workingMap.get(realm.realm_url) || 0; - let isIndexing = active.length > 0; - let statusClass = isIndexing ? 'indexing' : 'idle'; - - let activeJobsHtml = ''; - if (active.length > 0) { - activeJobsHtml = active - .map((job) => { - let changes = extractChanges(job); - let jobTypeLabel = job.job_type.replace('-', ' '); - let startedInfo = job.reservation_started - ? `started ${timeSince(job.reservation_started)}` - : `queued ${timeSince(job.created_at)}`; - let workerInfo = job.worker_id - ? ` (worker: ${escapeHtml(job.worker_id.substring(0, 8))})` - : ''; - - let changesHtml = ''; - if (changes.length > 0) { - let changesList = changes - .map((c) => `
  • ${escapeHtml(c)}
    • `) - .join(''); - changesHtml = ` -
    - ${changes.length} file${changes.length !== 1 ? 's' : ''} to process -
      ${changesList}
    -
    `; - } - - return ` -
    -
    - ${escapeHtml(jobTypeLabel)} - ${escapeHtml(startedInfo)}${escapeHtml(workerInfo)} -
    - ${changesHtml} - ${working > 0 ? `
    ${working} entries written to working index
    ` : ''} -
    `; - }) - .join(''); + return ` +
    +
    + +

    ${escapeHtml(state.realmURL)}

    +
    +
    + ${escapeHtml(state.jobType)} index + job #${state.jobId} · started ${timeSince(state.startedAt)} · ${durationMs(state.startedAt)} elapsed +
    +
    +
    + ${state.filesCompleted} / ${state.totalFiles} files (${pct}%) +
    +
    ${remaining} file${remaining !== 1 ? 's' : ''} remaining
    + ${ + remainingFiles.length > 0 + ? `
    + ${remaining} file${remaining !== 1 ? 's' : ''} left to index +
      ${remainingList}
    +
    ` + : '' } + ${ + state.completedFiles.length > 0 + ? `
    + ${state.filesCompleted} file${state.filesCompleted !== 1 ? 's' : ''} completed +
      ${completedList}
    +
    ` + : '' + } +
    `; +} - return ` -
    -
    - -

    ${escapeHtml(realm.realm_url)}

    -
    -
    - ${realm.total_entries} entries - ${realm.instances} instances - ${realm.files} files - ${realm.errors > 0 ? `${realm.errors} errors` : ''} -
    - ${activeJobsHtml} -
    `; - }) - .join(''); +function renderHistoryRow(state: RealmIndexingState): string { + let statsHtml = state.stats + ? Object.entries(state.stats) + .map(([k, v]) => `${k}: ${v}`) + .join(', ') + : ''; + return ` + + ${state.jobId} + ${escapeHtml(state.jobType)} + ${escapeHtml(state.realmURL)} + ${state.totalFiles} + ${durationMs(state.startedAt, state.lastUpdatedAt)} + ${timeSince(state.lastUpdatedAt)} + ${escapeHtml(statsHtml)} + `; +} - // Also show realms that have active jobs but no index entries yet - for (let [realmURL, jobs] of activeByRealm) { - if (!realmIndex.find((r) => r.realm_url === realmURL)) { - let working = workingMap.get(realmURL) || 0; - let jobsHtml = jobs - .map((job) => { - let jobTypeLabel = job.job_type.replace('-', ' '); - let startedInfo = job.reservation_started - ? `started ${timeSince(job.reservation_started)}` - : `queued ${timeSince(job.created_at)}`; - let changes = extractChanges(job); - let changesHtml = ''; - if (changes.length > 0) { - let changesList = changes - .map((c) => `
  • ${escapeHtml(c)}
    • `) - .join(''); - changesHtml = ` -
    - ${changes.length} file${changes.length !== 1 ? 's' : ''} to process -
      ${changesList}
    -
    `; - } - return ` -
    -
    - ${escapeHtml(jobTypeLabel)} - ${escapeHtml(startedInfo)} -
    - ${changesHtml} - ${working > 0 ? `
    ${working} entries written to working index
    ` : ''} -
    `; - }) - .join(''); +export interface DashboardSnapshot { + active: RealmIndexingState[]; + history: RealmIndexingState[]; +} - realmCards += ` -
    -
    - -

    ${escapeHtml(realmURL)}

    -
    -
    - 0 entries (new index) -
    - ${jobsHtml} -
    `; - } - } +export function renderIndexingDashboard(snapshot: DashboardSnapshot): string { + let { active, history } = snapshot; - // Recent completed jobs table - let recentJobsRows = recentJobs - .map((job) => { - let realmURL = extractRealmURL(job); - let stats = extractStats(job); - let statsHtml = stats - ? Object.entries(stats) - .map(([k, v]) => `${k}: ${v}`) - .join(', ') - : ''; - let statusClass = job.status === 'resolved' ? 'success' : 'failure'; - return ` - - ${job.id} - ${escapeHtml(job.job_type.replace('-', ' '))} - ${escapeHtml(realmURL)} - ${escapeHtml(job.status)} - ${duration(job.created_at, job.finished_at)} - ${job.finished_at ? timeSince(job.finished_at) : ''} - ${escapeHtml(statsHtml)} - `; - }) - .join(''); + let activeCards = active.map(renderActiveCard).join(''); + + let historyRows = history.map(renderHistoryRow).join(''); return ` @@ -377,7 +178,7 @@ function renderDashboard(data: Awaited>): str .summary-item.alert .value { color: #f0883e; } .realm-grid { display: grid; - grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(450px, 1fr)); gap: 16px; margin-bottom: 32px; } @@ -409,29 +210,12 @@ function renderDashboard(data: Awaited>): str 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } } - .realm-stats { + .job-info { display: flex; - gap: 12px; flex-wrap: wrap; - margin-bottom: 8px; - font-size: 13px; - color: #8b949e; - } - .realm-stats .stat strong { color: #e1e4e8; } - .realm-stats .stat.error strong { color: #f85149; } - .job-card { - background: #0d1117; - border: 1px solid #30363d; - border-radius: 6px; - padding: 10px 12px; - margin-top: 8px; - font-size: 13px; - } - .job-header { - display: flex; - justify-content: space-between; - align-items: center; gap: 8px; + align-items: center; + margin-bottom: 10px; } .job-type { font-weight: 600; @@ -439,12 +223,40 @@ function renderDashboard(data: Awaited>): str text-transform: capitalize; } .job-meta { color: #8b949e; font-size: 12px; } - .progress-info { - margin-top: 6px; + .progress-bar-container { + position: relative; + background: #21262d; + border-radius: 4px; + height: 24px; + margin-bottom: 6px; + overflow: hidden; + } + .progress-bar { + background: linear-gradient(90deg, #238636, #3fb950); + height: 100%; + border-radius: 4px; + transition: width 0.3s ease; + } + .progress-text { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + display: flex; + align-items: center; + justify-content: center; font-size: 12px; - color: #58a6ff; + font-weight: 600; + color: #fff; + text-shadow: 0 1px 2px rgba(0,0,0,0.5); + } + .remaining-count { + font-size: 13px; + color: #f0883e; + margin-bottom: 8px; } - details { margin-top: 8px; } + details { margin-top: 6px; } summary { cursor: pointer; color: #58a6ff; @@ -454,7 +266,7 @@ function renderDashboard(data: Awaited>): str .file-list { list-style: none; padding: 6px 0; - max-height: 200px; + max-height: 300px; overflow-y: auto; font-size: 12px; font-family: "SF Mono", Monaco, "Cascadia Code", monospace; @@ -464,6 +276,12 @@ function renderDashboard(data: Awaited>): str color: #c9d1d9; word-break: break-all; } + .file-list li.completed { + color: #3fb950; + } + .file-list li.completed::before { + content: "\\2713 "; + } table { width: 100%; border-collapse: collapse; @@ -481,16 +299,6 @@ function renderDashboard(data: Awaited>): str padding: 6px 12px; border-bottom: 1px solid #21262d; } - tr.failure td { color: #f85149; } - .status-badge { - display: inline-block; - padding: 2px 8px; - border-radius: 12px; - font-size: 11px; - font-weight: 600; - } - .status-badge.success { background: #238636; color: #fff; } - .status-badge.failure { background: #da3633; color: #fff; } .realm-url-cell { max-width: 300px; overflow: hidden; @@ -522,53 +330,49 @@ function renderDashboard(data: Awaited>): str
    -
    -
    ${activeJobs.length}
    +
    +
    ${active.length}
    Active Jobs
    -
    ${realmIndex.length}
    -
    Realms
    -
    -
    -
    ${realmIndex.reduce((s, r) => s + r.total_entries, 0)}
    -
    Total Entries
    +
    ${active.reduce((s, a) => s + (a.totalFiles - a.filesCompleted), 0)}
    +
    Files Remaining
    -
    ${realmIndex.reduce((s, r) => s + r.errors, 0)}
    -
    Total Errors
    +
    ${history.length}
    +
    Completed
    -

    Realms

    - ${realmCards.length > 0 ? `
    ${realmCards}
    ` : '
    No realms found
    '} +

    Active Indexing

    + ${activeCards.length > 0 ? `
    ${activeCards}
    ` : '
    No active indexing jobs
    '} -

    Recent Jobs

    +

    Recent Completed

    ${ - recentJobs.length > 0 + history.length > 0 ? `
    - + - + - ${recentJobsRows} + ${historyRows}
    IDJob Type RealmStatusFiles Duration Finished Stats
    ` - : '
    No recent jobs
    ' + : '
    No completed jobs yet (history is populated from events received since the worker manager started)
    ' }