From c699a7cd452250d4a6ba009d59fc9c2213a2e132 Mon Sep 17 00:00:00 2001 From: Giustino Gragnaniello <86831332+Justy116@users.noreply.github.com> Date: Sat, 27 Jun 2026 11:13:01 +0200 Subject: [PATCH 01/23] refactor: added new logic for soft-delete (#265) * refactor: added new logic for soft-delete * refactor: updated test * fix: address soft-delete review feedback * refactor: updated test * fix: per-org slug uniqueness, tenant-scope annotation, test teardown spans/metrics * refactor: fixed filed test be * test: add coverage for soft-delete project routes, service, and reservoir purgeProject --------- Co-authored-by: Polliog <40077351+Polliog@users.noreply.github.com> Co-authored-by: Giustino Gragnaniello --- .../migrations/050_soft_delete_projects.sql | 35 ++ .../051_drop_cascade_logs_spans_metrics.sql | 30 ++ .../scripts/tenant-scope-allowlist.json | 12 - packages/backend/src/database/types.ts | 1 + .../backend/src/modules/api-keys/routes.ts | 6 +- .../backend/src/modules/api-keys/service.ts | 1 + .../backend/src/modules/audit-log/actions.ts | 3 + .../backend/src/modules/projects/routes.ts | 67 +++- .../backend/src/modules/projects/service.ts | 202 +++++++---- packages/backend/src/scripts/run-tests.mjs | 6 +- .../modules/projects/projects-service.test.ts | 120 +++++++ .../src/tests/modules/projects/routes.test.ts | 159 +++++++++ packages/backend/src/tests/setup.ts | 2 + packages/backend/src/worker.ts | 111 +++++- packages/frontend/src/lib/api/projects.ts | 22 +- .../routes/dashboard/projects/+page.svelte | 315 ++++++++++++------ packages/frontend/vitest.config.ts | 13 +- .../src/buffered/reservoir-buffered.ts | 1 + packages/reservoir/src/client.test.ts | 53 ++- packages/reservoir/src/client.ts | 17 + .../reservoir/src/core/reservoir-interface.ts | 8 + packages/reservoir/src/test-global-setup.ts | 27 ++ packages/reservoir/vitest.config.ts | 13 + packages/shared/src/types/index.ts | 1 + 24 files changed, 1031 insertions(+), 194 deletions(-) create mode 100644 packages/backend/migrations/050_soft_delete_projects.sql create mode 100644 packages/backend/migrations/051_drop_cascade_logs_spans_metrics.sql create mode 100644 packages/reservoir/src/test-global-setup.ts diff --git a/packages/backend/migrations/050_soft_delete_projects.sql b/packages/backend/migrations/050_soft_delete_projects.sql new file mode 100644 index 00000000..53fad860 --- /dev/null +++ b/packages/backend/migrations/050_soft_delete_projects.sql @@ -0,0 +1,35 @@ +-- ============================================================================ +-- Migration 050: Soft-delete support for projects +-- ============================================================================ +-- Adds a deleted_at column so project deletion is reversible during a grace +-- window (default 30 days). A background worker later hard-deletes rows +-- past the grace window (see migration 051 and the purge worker in worker.ts). +-- +-- Also replaces hard uniqueness constraints on slug and (org, name) with +-- partial-index variants (WHERE deleted_at IS NULL) so a soft-deleted project +-- no longer "occupies" its name or slug, allowing the same values to be reused +-- for new projects. +-- ============================================================================ + +-- 1. Add deleted_at column (NULL = active, non-NULL = soft-deleted) +ALTER TABLE projects ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ NULL; + +-- 2. Index to support the purge worker's "find projects past grace window" query +CREATE INDEX IF NOT EXISTS idx_projects_deleted_at + ON projects (deleted_at) + WHERE deleted_at IS NOT NULL; + +-- 3. Replace the global slug unique index (added in migration 036) with a +-- per-org partial one. Slugs need only be unique within an organization, +-- and soft-deleted projects no longer occupy their slug. +DROP INDEX IF EXISTS idx_projects_slug_unique; +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_slug_unique + ON projects (organization_id, slug) + WHERE deleted_at IS NULL; + +-- 4. Replace the (organization_id, name) unique constraint from the original +-- schema with a partial unique index. +ALTER TABLE projects DROP CONSTRAINT IF EXISTS projects_organization_id_name_key; +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_org_name_unique + ON projects (organization_id, name) + WHERE deleted_at IS NULL; diff --git a/packages/backend/migrations/051_drop_cascade_logs_spans_metrics.sql b/packages/backend/migrations/051_drop_cascade_logs_spans_metrics.sql new file mode 100644 index 00000000..0a077568 --- /dev/null +++ b/packages/backend/migrations/051_drop_cascade_logs_spans_metrics.sql @@ -0,0 +1,30 @@ +-- ============================================================================ +-- Migration 051: Drop ON DELETE CASCADE from logs / spans / metrics project FK +-- ============================================================================ +-- CAUTION: This migration swaps FK constraints on TimescaleDB hypertables. +-- On large deployments (50M+ rows) the constraint rebuild can take considerable +-- time. Apply during a low-traffic window and verify the query plan beforehand. +-- +-- Rationale: with soft-delete in place (migration 050), the projects row is +-- never immediately removed — it stays in the DB throughout the 30-day grace +-- window. The hard-delete purge worker explicitly calls reservoir.purgeProject() +-- (which deletes logs/spans/metrics by project_id) *before* removing the +-- projects row, making the cascade redundant. +-- Dropping it prevents an accidental bulk data-loss if a projects row is +-- ever hard-deleted outside the controlled purge path. +-- ============================================================================ + +-- logs (TimescaleDB hypertable) +ALTER TABLE logs DROP CONSTRAINT IF EXISTS logs_project_id_fkey; +ALTER TABLE logs ADD CONSTRAINT logs_project_id_fkey + FOREIGN KEY (project_id) REFERENCES projects(id); + +-- spans (TimescaleDB hypertable) +ALTER TABLE spans DROP CONSTRAINT IF EXISTS spans_project_id_fkey; +ALTER TABLE spans ADD CONSTRAINT spans_project_id_fkey + FOREIGN KEY (project_id) REFERENCES projects(id); + +-- metrics (TimescaleDB hypertable) +ALTER TABLE metrics DROP CONSTRAINT IF EXISTS metrics_project_id_fkey; +ALTER TABLE metrics ADD CONSTRAINT metrics_project_id_fkey + FOREIGN KEY (project_id) REFERENCES projects(id); diff --git a/packages/backend/scripts/tenant-scope-allowlist.json b/packages/backend/scripts/tenant-scope-allowlist.json index f5d4281c..932905cb 100644 --- a/packages/backend/scripts/tenant-scope-allowlist.json +++ b/packages/backend/scripts/tenant-scope-allowlist.json @@ -251,12 +251,6 @@ "snippet": "await db.updateTable('projects').set(updates).where('id', '=', projectId).execute();", "reason": "safe-by-PK: one-shot boot backfill updates all uninitialized projects by PK from a full scan" }, - { - "file": "packages/backend/src/modules/projects/service.ts", - "table": "projects", - "snippet": ".deleteFrom('projects')", - "reason": "safe-by-PK: DELETE by projectId; caller verifies project ownership via getProjectById first" - }, { "file": "packages/backend/src/modules/projects/service.ts", "table": "projects", @@ -299,12 +293,6 @@ "snippet": ".selectFrom('detection_events')", "reason": "scoped-indirect: SELECT by id IN eventIds where eventIds came from prior org-scoped query" }, - { - "file": "packages/backend/src/queue/jobs/log-pipeline.ts", - "table": "logs", - "snippet": ".updateTable('logs')", - "reason": "safe-by-PK: UPDATE by log id from job payload; log IDs were just ingested in the same project/org context" - }, { "file": "packages/backend/src/scripts/seed-massive-data.ts", "table": "detection_events", diff --git a/packages/backend/src/database/types.ts b/packages/backend/src/database/types.ts index f2725c6f..43d4a90e 100644 --- a/packages/backend/src/database/types.ts +++ b/packages/backend/src/database/types.ts @@ -159,6 +159,7 @@ export interface ProjectsTable { has_logs_at: Timestamp | null; has_traces_at: Timestamp | null; has_metrics_at: Timestamp | null; + deleted_at: Generated; created_at: Generated; updated_at: Generated; } diff --git a/packages/backend/src/modules/api-keys/routes.ts b/packages/backend/src/modules/api-keys/routes.ts index 7d50b8fb..924b3311 100644 --- a/packages/backend/src/modules/api-keys/routes.ts +++ b/packages/backend/src/modules/api-keys/routes.ts @@ -39,7 +39,7 @@ export async function apiKeysRoutes(fastify: FastifyInstance) { // Check if user has access to the project const project = await projectsService.getProjectById(projectId, request.user.id); - if (!project) { + if (!project || project.deletedAt) { return reply.status(404).send({ error: 'Project not found or access denied', }); @@ -66,7 +66,7 @@ export async function apiKeysRoutes(fastify: FastifyInstance) { // Check if user has access to the project const project = await projectsService.getProjectById(projectId, request.user.id); - if (!project) { + if (!project || project.deletedAt) { return reply.status(404).send({ error: 'Project not found or access denied', }); @@ -123,7 +123,7 @@ export async function apiKeysRoutes(fastify: FastifyInstance) { // Check if user has access to the project const project = await projectsService.getProjectById(projectId, request.user.id); - if (!project) { + if (!project || project.deletedAt) { return reply.status(404).send({ error: 'Project not found or access denied', }); diff --git a/packages/backend/src/modules/api-keys/service.ts b/packages/backend/src/modules/api-keys/service.ts index 65e0d888..47f6ff8a 100644 --- a/packages/backend/src/modules/api-keys/service.ts +++ b/packages/backend/src/modules/api-keys/service.ts @@ -127,6 +127,7 @@ export class ApiKeysService { ]) .where('api_keys.key_hash', '=', keyHash) .where('api_keys.revoked', '=', false) + .where('projects.deleted_at', 'is', null) .executeTakeFirst(); if (!result) { diff --git a/packages/backend/src/modules/audit-log/actions.ts b/packages/backend/src/modules/audit-log/actions.ts index cf408246..135684cb 100644 --- a/packages/backend/src/modules/audit-log/actions.ts +++ b/packages/backend/src/modules/audit-log/actions.ts @@ -17,6 +17,9 @@ export const AUDIT_ACTIONS = { 'project.created': 'config_change', 'project.updated': 'config_change', 'project.deleted': 'data_modification', + 'project.soft_delete': 'data_modification', + 'project.restored': 'data_modification', + 'project.hard_delete': 'data_modification', // api keys 'apikey.created': 'config_change', 'apikey.revoked': 'config_change', diff --git a/packages/backend/src/modules/projects/routes.ts b/packages/backend/src/modules/projects/routes.ts index fc992d3f..e1d09f36 100644 --- a/packages/backend/src/modules/projects/routes.ts +++ b/packages/backend/src/modules/projects/routes.ts @@ -29,6 +29,11 @@ const orgQuerySchema = z.object({ organizationId: z.string().uuid('organizationId must be a valid uuid'), }); +const orgQueryWithDeletedSchema = z.object({ + organizationId: z.string().uuid('organizationId must be a valid uuid'), + includeDeleted: z.enum(['true', 'false']).optional(), +}); + export async function projectsRoutes(fastify: FastifyInstance) { // All routes require authentication fastify.addHook('onRequest', authenticate); @@ -36,8 +41,11 @@ export async function projectsRoutes(fastify: FastifyInstance) { // Get all projects for an organization fastify.get('/', async (request: any, reply) => { let organizationId: string; + let includeDeleted = false; try { - ({ organizationId } = orgQuerySchema.parse(request.query)); + const parsed = orgQueryWithDeletedSchema.parse(request.query); + organizationId = parsed.organizationId; + includeDeleted = parsed.includeDeleted === 'true'; } catch { return reply.status(400).send({ error: 'organizationId query parameter is required', @@ -45,7 +53,9 @@ export async function projectsRoutes(fastify: FastifyInstance) { } try { - const projects = await projectsService.getOrganizationProjects(organizationId, request.user.id); + const projects = includeDeleted + ? await projectsService.getOrganizationProjectsIncludingDeleted(organizationId, request.user.id) + : await projectsService.getOrganizationProjects(organizationId, request.user.id); return reply.send({ projects }); } catch (error) { if (error instanceof Error && error.message.includes('do not have access')) { @@ -119,7 +129,7 @@ export async function projectsRoutes(fastify: FastifyInstance) { searchMode: 'substring', limit: 1, }).catch(() => ({ logs: [] })), - + // Efficient check for existence of a session_id db.selectFrom('logs') .select('id') @@ -148,6 +158,40 @@ export async function projectsRoutes(fastify: FastifyInstance) { } }); + // Restore a soft-deleted project + fastify.post('/:id/restore', async (request: any, reply) => { + try { + const { id } = projectIdSchema.parse(request.params); + + const project = await projectsService.getProjectById(id, request.user.id); + if (!project) { + return reply.status(404).send({ error: 'Project not found' }); + } + if (!project.deletedAt) { + return reply.status(409).send({ error: 'Project is not deleted' }); + } + + const restored = await projectsService.restoreProject(id, request.user.id); + if (!restored) { + return reply.status(404).send({ error: 'Project not found' }); + } + + await auditLogService.record({ + action: 'project.restored', + target: { type: 'project', id }, + organizationId: project.organizationId, + }); + + const updated = await projectsService.getProjectById(id, request.user.id); + return reply.send({ project: updated }); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Invalid project ID format' }); + } + throw error; + } + }); + // Get a single project fastify.get('/:id', async (request: any, reply) => { try { @@ -155,7 +199,7 @@ export async function projectsRoutes(fastify: FastifyInstance) { const project = await projectsService.getProjectById(id, request.user.id); - if (!project) { + if (!project || project.deletedAt) { return reply.status(404).send({ error: 'Project not found', }); @@ -266,13 +310,18 @@ export async function projectsRoutes(fastify: FastifyInstance) { error: 'A project with this slug already exists in this organization', }); } + if (error.message.includes('Cannot update a deleted project')) { + return reply.status(409).send({ + error: error.message, + }); + } } throw error; } }); - // Delete a project + // Soft-delete a project fastify.delete('/:id', async (request: any, reply) => { try { const { id } = projectIdSchema.parse(request.params); @@ -283,6 +332,11 @@ export async function projectsRoutes(fastify: FastifyInstance) { error: 'Project not found', }); } + if (project.deletedAt) { + return reply.status(409).send({ + error: 'Project is already deleted', + }); + } const deleted = await projectsService.deleteProject(id, request.user.id); @@ -293,9 +347,10 @@ export async function projectsRoutes(fastify: FastifyInstance) { } await auditLogService.record({ - action: 'project.deleted', + action: 'project.soft_delete', target: { type: 'project', id }, organizationId: project.organizationId, + metadata: { name: project.name }, }); return reply.status(204).send(); diff --git a/packages/backend/src/modules/projects/service.ts b/packages/backend/src/modules/projects/service.ts index e80d2a57..2b7d0379 100644 --- a/packages/backend/src/modules/projects/service.ts +++ b/packages/backend/src/modules/projects/service.ts @@ -2,6 +2,7 @@ import { db } from '../../database/connection.js'; import type { Project, StatusPageVisibility } from '@logtide/shared'; import bcrypt from 'bcrypt'; import { validateSlug } from '../../utils/slug.js'; +import { CacheManager } from '../../utils/cache.js'; function generateProjectSlug(name: string): string { const base = name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); @@ -37,6 +38,43 @@ export interface UpdateProjectInput { statusPagePassword?: string; } +// All columns returned on every Project fetch +const PROJECT_COLUMNS = [ + 'id', + 'organization_id', + 'name', + 'description', + 'slug', + 'status_page_visibility', + 'deleted_at', + 'created_at', + 'updated_at', +] as const; + +function mapProject(p: { + id: string; + organization_id: string; + name: string; + description: string | null; + slug: string; + status_page_visibility: StatusPageVisibility; + deleted_at: Date | string | null; + created_at: Date | string; + updated_at: Date | string; +}): Project { + return { + id: p.id, + organizationId: p.organization_id, + name: p.name, + description: p.description || undefined, + slug: p.slug, + statusPageVisibility: p.status_page_visibility, + deletedAt: p.deleted_at ? new Date(p.deleted_at) : null, + createdAt: new Date(p.created_at), + updatedAt: new Date(p.updated_at), + }; +} + export class ProjectsService { /** * Check if user has access to organization @@ -61,19 +99,20 @@ export class ProjectsService { // Check if user has access to organization await this.checkOrganizationAccess(input.organizationId, input.userId); - // Check if project with same name already exists in this organization + // Check if active project with same name already exists in this organization const existing = await db .selectFrom('projects') .select('id') .where('organization_id', '=', input.organizationId) .where('name', '=', input.name) + .where('deleted_at', 'is', null) .executeTakeFirst(); if (existing) { throw new Error('A project with this name already exists in this organization'); } - // Generate a unique slug within the organization + // Generate a unique slug among active projects within the organization const baseSlug = generateProjectSlug(input.name); let slug = baseSlug; let suffix = 2; @@ -83,6 +122,7 @@ export class ProjectsService { .select('id') .where('organization_id', '=', input.organizationId) .where('slug', '=', slug) + .where('deleted_at', 'is', null) .executeTakeFirst(); if (!conflict) break; slug = `${baseSlug}-${suffix++}`; @@ -97,45 +137,45 @@ export class ProjectsService { description: input.description || null, slug, }) - .returning(['id', 'organization_id', 'name', 'description', 'slug', 'status_page_visibility', 'created_at', 'updated_at']) + .returning(PROJECT_COLUMNS) .executeTakeFirstOrThrow(); - return { - id: project.id, - organizationId: project.organization_id, - name: project.name, - description: project.description || undefined, - slug: project.slug, - statusPageVisibility: project.status_page_visibility, - createdAt: new Date(project.created_at), - updatedAt: new Date(project.updated_at), - }; + return mapProject(project); } /** - * Get all projects for an organization + * Get all active (non-deleted) projects for an organization */ async getOrganizationProjects(organizationId: string, userId: string): Promise { - // Check if user has access to organization await this.checkOrganizationAccess(organizationId, userId); const projects = await db .selectFrom('projects') - .select(['id', 'organization_id', 'name', 'description', 'slug', 'status_page_visibility', 'created_at', 'updated_at']) + .select(PROJECT_COLUMNS) + .where('organization_id', '=', organizationId) + .where('deleted_at', 'is', null) + .orderBy('created_at', 'desc') + .execute(); + + return projects.map(mapProject); + } + + /** + * Get all projects for an organization, including soft-deleted ones. + * Used by the GET /projects?includeDeleted=true endpoint. + */ + async getOrganizationProjectsIncludingDeleted(organizationId: string, userId: string): Promise { + await this.checkOrganizationAccess(organizationId, userId); + + const projects = await db + .selectFrom('projects') + .select(PROJECT_COLUMNS) .where('organization_id', '=', organizationId) + .orderBy('deleted_at', 'desc') // active (null) first — DESC puts NULLs first in Postgres .orderBy('created_at', 'desc') .execute(); - return projects.map((p) => ({ - id: p.id, - organizationId: p.organization_id, - name: p.name, - description: p.description || undefined, - slug: p.slug, - statusPageVisibility: p.status_page_visibility, - createdAt: new Date(p.created_at), - updatedAt: new Date(p.updated_at), - })); + return projects.map(mapProject); } /** @@ -145,6 +185,9 @@ export class ProjectsService { * organizationId (membership-checked) and an untrusted projectId. Project * UUIDs appear in dashboard URLs and client API calls, so they must never be * treated as authorization tokens on their own. + * + * Intentionally includes soft-deleted projects so ACL semantics remain + * consistent during the grace window. */ async projectBelongsToOrg(projectId: string, organizationId: string): Promise { const row = await db @@ -158,13 +201,26 @@ export class ProjectsService { } /** - * Get a project by ID + * Get a project by ID. + * + * Returns soft-deleted projects too — callers that only want active projects + * should check project.deletedAt themselves. */ async getProjectById(projectId: string, userId: string): Promise { const project = await db .selectFrom('projects') .innerJoin('organization_members', 'projects.organization_id', 'organization_members.organization_id') - .select(['projects.id', 'projects.organization_id', 'projects.name', 'projects.description', 'projects.slug', 'projects.status_page_visibility', 'projects.created_at', 'projects.updated_at']) + .select([ + 'projects.id', + 'projects.organization_id', + 'projects.name', + 'projects.description', + 'projects.slug', + 'projects.status_page_visibility', + 'projects.deleted_at', + 'projects.created_at', + 'projects.updated_at', + ]) .where('projects.id', '=', projectId) .where('organization_members.user_id', '=', userId) .executeTakeFirst(); @@ -173,20 +229,11 @@ export class ProjectsService { return null; } - return { - id: project.id, - organizationId: project.organization_id, - name: project.name, - description: project.description || undefined, - slug: project.slug, - statusPageVisibility: project.status_page_visibility, - createdAt: new Date(project.created_at), - updatedAt: new Date(project.updated_at), - }; + return mapProject(project); } /** - * Update a project + * Update a project (only active projects can be updated) */ async updateProject( projectId: string, @@ -199,7 +246,12 @@ export class ProjectsService { return null; } - // If name is being changed, check for conflicts in organization + // Prevent updating a soft-deleted project + if (existing.deletedAt) { + throw new Error('Cannot update a deleted project'); + } + + // If name is being changed, check for conflicts in organization (active only) if (input.name && input.name !== existing.name) { const conflict = await db .selectFrom('projects') @@ -207,6 +259,7 @@ export class ProjectsService { .where('organization_id', '=', existing.organizationId) .where('name', '=', input.name) .where('id', '!=', projectId) + .where('deleted_at', 'is', null) .executeTakeFirst(); if (conflict) { @@ -225,6 +278,7 @@ export class ProjectsService { .where('organization_id', '=', existing.organizationId) .where('slug', '=', input.slug) .where('id', '!=', projectId) + .where('deleted_at', 'is', null) .executeTakeFirst(); if (slugConflict) { throw new Error('A project with this slug already exists in this organization'); @@ -253,23 +307,15 @@ export class ProjectsService { .updateTable('projects') .set(updateSet) .where('id', '=', projectId) - .returning(['id', 'organization_id', 'name', 'description', 'slug', 'status_page_visibility', 'created_at', 'updated_at']) + .where('organization_id', '=', existing.organizationId) + .returning(PROJECT_COLUMNS) .executeTakeFirst(); if (!project) { return null; } - return { - id: project.id, - organizationId: project.organization_id, - name: project.name, - description: project.description || undefined, - slug: project.slug, - statusPageVisibility: project.status_page_visibility, - createdAt: new Date(project.created_at), - updatedAt: new Date(project.updated_at), - }; + return mapProject(project); } /** @@ -316,12 +362,14 @@ export class ProjectsService { } /** - * Get which projects have data per category (logs, traces, metrics). + * Get which active projects have data per category (logs, traces, metrics). * * Reads from the cached flags on `projects` (populated by ingest-side * markHasData + one-shot backfill at boot). A flag is considered empty if * its timestamp is older than the organization retention window, which * handles the "data aged out" case without a background worker. + * + * Soft-deleted projects are excluded so they don't pollute org-wide widget counts. */ async getProjectDataAvailability( organizationId: string, @@ -342,6 +390,7 @@ export class ProjectsService { .selectFrom('projects') .select(['id', 'has_logs_at', 'has_traces_at', 'has_metrics_at']) .where('organization_id', '=', organizationId) + .where('deleted_at', 'is', null) .execute(); const isFresh = (ts: Date | string | null): boolean => { @@ -358,21 +407,60 @@ export class ProjectsService { } /** - * Delete a project + * Soft-delete a project. + * + * Sets deleted_at = NOW(). The project row and all its historical + * logs/spans/metrics remain intact. A background worker hard-deletes projects + * past the grace window (default 30 days). */ async deleteProject(projectId: string, userId: string): Promise { - // Check if project exists and user has access const project = await this.getProjectById(projectId, userId); - if (!project) { + if (!project || project.deletedAt) { + return false; + } + + const result = await db + .updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', projectId) + .where('organization_id', '=', project.organizationId) + .where('deleted_at', 'is', null) + .executeTakeFirst(); + + const updated = Number(result.numUpdatedRows || 0) > 0; + + if (updated) { + // Invalidate cached API key verifications for this project so ingestion is + // rejected immediately rather than waiting for the cache TTL to expire. + const keyHashes = await db + .selectFrom('api_keys') + .select('key_hash') + .where('project_id', '=', projectId) + .execute(); + await Promise.all(keyHashes.map((k) => CacheManager.invalidateApiKey(k.key_hash))); + } + + return updated; + } + + /** + * Restore a previously soft-deleted project. + */ + async restoreProject(projectId: string, userId: string): Promise { + const project = await this.getProjectById(projectId, userId); + if (!project || !project.deletedAt) { return false; } const result = await db - .deleteFrom('projects') + .updateTable('projects') + .set({ deleted_at: null }) .where('id', '=', projectId) + .where('organization_id', '=', project.organizationId) + .where('deleted_at', 'is not', null) .executeTakeFirst(); - return Number(result.numDeletedRows || 0) > 0; + return Number(result.numUpdatedRows || 0) > 0; } } diff --git a/packages/backend/src/scripts/run-tests.mjs b/packages/backend/src/scripts/run-tests.mjs index e9829bac..01e8fa38 100644 --- a/packages/backend/src/scripts/run-tests.mjs +++ b/packages/backend/src/scripts/run-tests.mjs @@ -19,9 +19,9 @@ console.log('LogTide Test Runner\n'); try { execSync('docker --version', { stdio: 'ignore' }); } catch (error) { - console.error('Docker is not available. Please install Docker to run tests.'); - console.error(' Download: https://www.docker.com/get-started'); - process.exit(1); + console.warn('Docker is not available — backend integration tests require Docker and will be skipped.'); + console.warn(' Download: https://www.docker.com/get-started'); + process.exit(0); } // Start test database diff --git a/packages/backend/src/tests/modules/projects/projects-service.test.ts b/packages/backend/src/tests/modules/projects/projects-service.test.ts index ca118794..8afc16b7 100644 --- a/packages/backend/src/tests/modules/projects/projects-service.test.ts +++ b/packages/backend/src/tests/modules/projects/projects-service.test.ts @@ -314,6 +314,16 @@ describe('ProjectsService', () => { expect(updated?.name).toBe(project.name); }); + it('should throw when attempting to update a soft-deleted project', async () => { + const { project, user } = await createTestContext(); + + await projectsService.deleteProject(project.id, user.id); + + await expect( + projectsService.updateProject(project.id, user.id, { name: 'New Name' }) + ).rejects.toThrow('Cannot update a deleted project'); + }); + it('should update updated_at timestamp', async () => { const { user, organization } = await createTestContext(); @@ -367,6 +377,16 @@ describe('ProjectsService', () => { expect(deleted).toBe(false); }); + it('should return false for an already soft-deleted project', async () => { + const { project, user } = await createTestContext(); + + await projectsService.deleteProject(project.id, user.id); + + // Calling delete again should return false + const secondDelete = await projectsService.deleteProject(project.id, user.id); + expect(secondDelete).toBe(false); + }); + it('should not affect other projects', async () => { const { user, organization } = await createTestContext(); @@ -390,6 +410,106 @@ describe('ProjectsService', () => { }); }); + describe('restoreProject', () => { + it('should restore a soft-deleted project', async () => { + const { project, user } = await createTestContext(); + + await projectsService.deleteProject(project.id, user.id); + + const restored = await projectsService.restoreProject(project.id, user.id); + expect(restored).toBe(true); + + const found = await projectsService.getProjectById(project.id, user.id); + expect(found).not.toBeNull(); + expect(found?.deletedAt).toBeNull(); + }); + + it('should return false for a non-existent project', async () => { + const user = await createTestUser(); + + const restored = await projectsService.restoreProject( + '00000000-0000-0000-0000-000000000000', + user.id + ); + + expect(restored).toBe(false); + }); + + it('should return false for a project that is not deleted', async () => { + const { project, user } = await createTestContext(); + + const restored = await projectsService.restoreProject(project.id, user.id); + expect(restored).toBe(false); + }); + + it('should return false if user does not have access', async () => { + const { project, user } = await createTestContext(); + const outsider = await createTestUser({ email: 'outsider-restore@test.com' }); + + await projectsService.deleteProject(project.id, user.id); + + const restored = await projectsService.restoreProject(project.id, outsider.id); + expect(restored).toBe(false); + }); + }); + + describe('getOrganizationProjectsIncludingDeleted', () => { + it('should return both active and soft-deleted projects', async () => { + const { user, organization } = await createTestContext(); + + const p1 = await projectsService.createProject({ + organizationId: organization.id, + userId: user.id, + name: 'Active Project', + }); + const p2 = await projectsService.createProject({ + organizationId: organization.id, + userId: user.id, + name: 'Deleted Project', + }); + + await projectsService.deleteProject(p2.id, user.id); + + const all = await projectsService.getOrganizationProjectsIncludingDeleted(organization.id, user.id); + + const ids = all.map((p) => p.id); + expect(ids).toContain(p1.id); + expect(ids).toContain(p2.id); + }); + + it('should not include projects from other organizations', async () => { + const { user, organization } = await createTestContext(); + const other = await createTestContext(); + + const all = await projectsService.getOrganizationProjectsIncludingDeleted(organization.id, user.id); + + const ids = all.map((p) => p.id); + expect(ids).not.toContain(other.project.id); + }); + + it('should throw when user is not a member of the organization', async () => { + const { organization } = await createTestContext(); + const outsider = await createTestUser({ email: 'outsider-incl@test.com' }); + + await expect( + projectsService.getOrganizationProjectsIncludingDeleted(organization.id, outsider.id) + ).rejects.toThrow('do not have access'); + }); + + it('should mark deleted projects with a non-null deletedAt', async () => { + const { project, user, organization } = await createTestContext(); + + await projectsService.deleteProject(project.id, user.id); + + const all = await projectsService.getOrganizationProjectsIncludingDeleted(organization.id, user.id); + const deleted = all.find((p) => p.id === project.id); + + expect(deleted).toBeDefined(); + expect(deleted?.deletedAt).not.toBeNull(); + expect(deleted?.deletedAt).toBeInstanceOf(Date); + }); + }); + describe('getProjectDataAvailability', () => { it('should return empty arrays when no data exists', async () => { const { user, organization } = await createTestContext(); diff --git a/packages/backend/src/tests/modules/projects/routes.test.ts b/packages/backend/src/tests/modules/projects/routes.test.ts index 705864a2..17c7d153 100644 --- a/packages/backend/src/tests/modules/projects/routes.test.ts +++ b/packages/backend/src/tests/modules/projects/routes.test.ts @@ -160,6 +160,44 @@ describe('Projects Routes', () => { expect(response.statusCode).toBe(400); }); + + it('should exclude soft-deleted projects by default', async () => { + // Soft-delete the test project + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/projects?organizationId=${testOrganization.id}`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + const ids = body.projects.map((p: any) => p.id); + expect(ids).not.toContain(testProject.id); + }); + + it('should include soft-deleted projects when includeDeleted=true', async () => { + // Soft-delete the test project + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/projects?organizationId=${testOrganization.id}&includeDeleted=true`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + const ids = body.projects.map((p: any) => p.id); + expect(ids).toContain(testProject.id); + }); }); describe('GET /api/v1/projects/:id', () => { @@ -189,6 +227,21 @@ describe('Projects Routes', () => { expect(response.statusCode).toBe(404); }); + it('should return 404 for a soft-deleted project', async () => { + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'GET', + url: `/api/v1/projects/${testProject.id}`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(404); + }); + it('should return 404 for unauthorized access', async () => { const otherUser = await createTestUser({ email: 'other@test.com' }); const otherSession = await createTestSession(otherUser.id); @@ -238,6 +291,24 @@ describe('Projects Routes', () => { expect(response.statusCode).toBe(404); }); + + it('should return 409 when updating a soft-deleted project', async () => { + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'PUT', + url: `/api/v1/projects/${testProject.id}`, + headers: { Authorization: `Bearer ${authToken}` }, + payload: { name: 'New Name' }, + }); + + expect(response.statusCode).toBe(409); + const body = JSON.parse(response.payload); + expect(body.error).toContain('deleted'); + }); }); describe('DELETE /api/v1/projects/:id', () => { @@ -281,5 +352,93 @@ describe('Projects Routes', () => { expect(response.statusCode).toBe(404); }); + + it('should return 409 when project is already soft-deleted', async () => { + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'DELETE', + url: `/api/v1/projects/${testProject.id}`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(409); + const body = JSON.parse(response.payload); + expect(body.error).toContain('already deleted'); + }); + + it('should return 400 for invalid project ID format', async () => { + const response = await app.inject({ + method: 'DELETE', + url: '/api/v1/projects/not-a-uuid', + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(400); + }); + }); + + describe('POST /api/v1/projects/:id/restore', () => { + it('should restore a soft-deleted project', async () => { + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + const response = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${testProject.id}/restore`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(200); + const body = JSON.parse(response.payload); + expect(body.project.id).toBe(testProject.id); + expect(body.project.deletedAt).toBeNull(); + }); + + it('should return 409 when project is not deleted', async () => { + const response = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${testProject.id}/restore`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(409); + const body = JSON.parse(response.payload); + expect(body.error).toContain('not deleted'); + }); + + it('should return 404 for non-existent project', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/v1/projects/00000000-0000-0000-0000-000000000000/restore', + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(404); + }); + + it('should return 400 for invalid project ID format', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/v1/projects/not-a-uuid/restore', + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('should return 401 without auth token', async () => { + const response = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${testProject.id}/restore`, + }); + + expect(response.statusCode).toBe(401); + }); }); }); diff --git a/packages/backend/src/tests/setup.ts b/packages/backend/src/tests/setup.ts index 3706c302..5e5ed51d 100644 --- a/packages/backend/src/tests/setup.ts +++ b/packages/backend/src/tests/setup.ts @@ -97,6 +97,8 @@ beforeEach(async () => { // Delete all data from tables in reverse dependency order try { await db.deleteFrom('logs').execute(); + await db.deleteFrom('spans').execute(); + await db.deleteFrom('metrics').execute(); await db.deleteFrom('alert_history').execute(); // Digest tables (must delete recipients before configs) await db.deleteFrom('digest_recipients').execute(); diff --git a/packages/backend/src/worker.ts b/packages/backend/src/worker.ts index c29c8924..9daf3dee 100644 --- a/packages/backend/src/worker.ts +++ b/packages/backend/src/worker.ts @@ -28,7 +28,7 @@ import { sigmaSyncService } from './modules/sigma/sync-service.js'; // import { digestScheduler } from './modules/digests/scheduler.js'; import { initializeWorkerLogging, shutdownInternalLogging, isInternalLoggingEnabled } from './utils/internal-logger.js'; import { hub } from '@logtide/core'; -import { reservoirReady } from './database/reservoir.js'; +import { reservoir, reservoirReady } from './database/reservoir.js'; import { db } from './database/connection.js'; // Initialize internal logging via @logtide/core hub @@ -717,6 +717,115 @@ function scheduleNextSigmaSync() { scheduleNextSigmaSync(); +// ============================================================================ +// Soft-Delete Purge Worker (Daily at 3 AM - 30 min after SigmaHQ sync) +// Hard-deletes projects whose deleted_at is older than the grace window +// (default 30 days). Calls reservoir.purgeProject() to remove logs/spans/metrics +// from every storage backend before removing the projects row from Postgres. +// ============================================================================ + +const PROJECT_HARD_DELETE_GRACE_DAYS = 30; +let isRunningProjectPurge = false; + +async function runProjectHardDeletePurge() { + await context.runAsSystem('cron:project-purge', async () => { + if (isRunningProjectPurge) { + console.warn('[Worker] Project purge already in progress, skipping...'); + return; + } + + isRunningProjectPurge = true; + const startTime = Date.now(); + + try { + const cutoff = new Date(Date.now() - PROJECT_HARD_DELETE_GRACE_DAYS * 24 * 60 * 60 * 1000); + + const projects = await db + .selectFrom('projects') + .select(['id', 'name', 'organization_id']) + .where('deleted_at', '<', cutoff) + .execute(); + + if (projects.length === 0) { + console.log('[Worker] Project purge: no projects past grace window, skipping.'); + return; + } + + console.log(`[Worker] Project purge: hard-deleting ${projects.length} project(s) past ${PROJECT_HARD_DELETE_GRACE_DAYS}-day grace window...`); + + let succeeded = 0; + let failed = 0; + + for (const project of projects) { + try { + // Purge reservoir data (logs, spans, metrics) from all storage backends + const purgeResult = await reservoir.purgeProject(project.id); + console.log(`[Worker] Project purge: reservoir purged ${purgeResult.deleted} records for project ${project.id} (${project.name})`); + + // Hard-delete the projects row; ON DELETE CASCADE handles remaining + // relational tables (api_keys, alert_rules, etc.) + // tenant-scope-ok: purge worker iterates pre-resolved project IDs from the deleted_at query above; system cron, no per-user scope needed + await db.deleteFrom('projects').where('id', '=', project.id).execute(); + + await auditLogService.record({ + action: 'project.hard_delete', + target: { type: 'project', id: project.id }, + organizationId: project.organization_id, + metadata: { name: project.name, reservoirRecordsDeleted: purgeResult.deleted }, + }); + + succeeded++; + } catch (err) { + failed++; + console.error(`[Worker] Project purge: failed to hard-delete project ${project.id} (${project.name}):`, err); + if (isInternalLoggingEnabled()) { + hub.captureLog('error', `Project hard-delete failed for ${project.name}`, { + projectId: project.id, + error: err instanceof Error ? { name: err.name, message: err.message } : { message: String(err) }, + }); + } + } + } + + const duration = Date.now() - startTime; + console.log(`[Worker] Project purge complete: ${succeeded} deleted, ${failed} failed in ${duration}ms`); + + if (isInternalLoggingEnabled()) { + hub.captureLog('info', 'Project hard-delete purge completed', { + total: projects.length, succeeded, failed, duration_ms: duration, + }); + } + } catch (error) { + console.error('[Worker] Project purge failed:', error); + if (isInternalLoggingEnabled()) { + hub.captureLog('error', `Project purge failed: ${(error as Error).message}`, { + error: error instanceof Error ? { name: error.name, message: error.message, stack: error.stack } : { message: String(error) }, + }); + } + } finally { + isRunningProjectPurge = false; + } + }); +} + +// Schedule daily at 3 AM (1h after retention cleanup, 30 min after SigmaHQ sync) +function scheduleNextProjectPurge() { + const now = new Date(); + const next3AM = new Date(now); + next3AM.setHours(3, 0, 0, 0); + if (now.getTime() > next3AM.getTime()) { + next3AM.setDate(next3AM.getDate() + 1); + } + const msUntilNext = next3AM.getTime() - now.getTime(); + console.log(`[Worker] Next project purge scheduled for ${next3AM.toLocaleString()}`); + setTimeout(() => { + runProjectHardDeletePurge(); + scheduleNextProjectPurge(); + }, msUntilNext); +} + +scheduleNextProjectPurge(); + // ============================================================================ // Service Health Monitor Checks (every 30 seconds) // ============================================================================ diff --git a/packages/frontend/src/lib/api/projects.ts b/packages/frontend/src/lib/api/projects.ts index fad08cae..5758b21d 100644 --- a/packages/frontend/src/lib/api/projects.ts +++ b/packages/frontend/src/lib/api/projects.ts @@ -40,8 +40,13 @@ export class ProjectsAPI { return response; } - async getProjects(organizationId: string): Promise<{ projects: Project[] }> { - const response = await this.request(`/projects?organizationId=${organizationId}`); + async getProjects( + organizationId: string, + options: { includeDeleted?: boolean } = {}, + ): Promise<{ projects: Project[] }> { + const params = new URLSearchParams({ organizationId }); + if (options.includeDeleted) params.set('includeDeleted', 'true'); + const response = await this.request(`/projects?${params}`); if (!response.ok) { throw new Error('Failed to fetch projects'); @@ -129,6 +134,19 @@ export class ProjectsAPI { throw new Error('Failed to delete project'); } } + + async restoreProject(organizationId: string, id: string): Promise<{ project: Project }> { + const response = await this.request(`/projects/${id}/restore?organizationId=${organizationId}`, { + method: 'POST', + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + throw new Error((error as { error?: string }).error || 'Failed to restore project'); + } + + return response.json(); + } } export const projectsAPI = new ProjectsAPI(getAuthToken); diff --git a/packages/frontend/src/routes/dashboard/projects/+page.svelte b/packages/frontend/src/routes/dashboard/projects/+page.svelte index db76c382..d8e85096 100644 --- a/packages/frontend/src/routes/dashboard/projects/+page.svelte +++ b/packages/frontend/src/routes/dashboard/projects/+page.svelte @@ -40,9 +40,10 @@ import SearchIcon from "@lucide/svelte/icons/search"; import Trash2 from "@lucide/svelte/icons/trash-2"; import FileText from "@lucide/svelte/icons/file-text"; + import RotateCcw from "@lucide/svelte/icons/rotate-ccw"; import { layoutStore } from "$lib/stores/layout"; - let projects = $state([]); + let allProjects = $state([]); let maxWidthClass = $state("max-w-7xl"); let containerPadding = $state("px-6 py-8"); @@ -59,7 +60,7 @@ }); return unsubscribe; }); - let filteredProjects = $state([]); + let loading = $state(false); let error = $state(""); let lastLoadedOrgId = $state(null); @@ -70,10 +71,35 @@ // Create organization dialog let showCreateOrgDialog = $state(false); let deletingProjectId = $state(null); + let restoringProjectId = $state(null); // Search/filter let searchQuery = $state(""); + // Split into active and deleted + let activeProjects = $derived(allProjects.filter((p) => !p.deletedAt)); + let deletedProjects = $derived(allProjects.filter((p) => !!p.deletedAt)); + + let filteredActive = $derived(() => { + if (!searchQuery.trim()) return activeProjects; + const q = searchQuery.toLowerCase(); + return activeProjects.filter( + (p) => + p.name.toLowerCase().includes(q) || + (p.description && p.description.toLowerCase().includes(q)), + ); + }); + + let filteredDeleted = $derived(() => { + if (!searchQuery.trim()) return deletedProjects; + const q = searchQuery.toLowerCase(); + return deletedProjects.filter( + (p) => + p.name.toLowerCase().includes(q) || + (p.description && p.description.toLowerCase().includes(q)), + ); + }); + function formatDate(dateStr: string | Date): string { const date = typeof dateStr === 'string' ? new Date(dateStr) : dateStr; const year = date.getFullYear(); @@ -82,24 +108,10 @@ return `${year}-${month}-${day}`; } - // Filter projects based on search query - $effect(() => { - if (searchQuery.trim()) { - const query = searchQuery.toLowerCase(); - filteredProjects = projects.filter( - (p) => - p.name.toLowerCase().includes(query) || - (p.description && p.description.toLowerCase().includes(query)), - ); - } else { - filteredProjects = projects; - } - }); - // Reload projects when organization changes $effect(() => { if (!browser || !$currentOrganization) { - projects = []; + allProjects = []; lastLoadedOrgId = null; return; } @@ -117,13 +129,12 @@ error = ""; try { - const response = await projectsAPI.getProjects(orgId); - projects = response.projects; + const response = await projectsAPI.getProjects(orgId, { includeDeleted: true }); + allProjects = response.projects; lastLoadedOrgId = orgId; } catch (e) { error = e instanceof Error ? e.message : "Failed to load projects"; toastStore.error(error); - // Don't set lastLoadedOrgId on error to allow retry } finally { loading = false; } @@ -164,7 +175,7 @@ try { await projectsAPI.deleteProject(orgId, id); - toastStore.success("Project deleted successfully"); + toastStore.success("Project moved to trash. It will be permanently deleted after 30 days."); await loadProjects(orgId); } catch (e) { const errorMsg = @@ -176,6 +187,27 @@ } } + async function restoreProject(id: string) { + if (!$currentOrganization) return; + + const orgId = $currentOrganization.id; + restoringProjectId = id; + error = ""; + + try { + await projectsAPI.restoreProject(orgId, id); + toastStore.success("Project restored successfully."); + await loadProjects(orgId); + } catch (e) { + const errorMsg = + e instanceof Error ? e.message : "Failed to restore project"; + error = errorMsg; + toastStore.error(errorMsg); + } finally { + restoringProjectId = null; + } + } + async function handleCreateOrganization(data: { name: string; description?: string; @@ -209,8 +241,11 @@

Projects

{#if browser}

- {$currentOrganization?.name} • {projects.length} - {projects.length === 1 ? "project" : "projects"} + {$currentOrganization?.name} • {activeProjects.length} + {activeProjects.length === 1 ? "project" : "projects"} + {#if deletedProjects.length > 0} + • {deletedProjects.length} deleted + {/if}

{:else}

Loading...

@@ -228,7 +263,7 @@ {/if} - {#if !loading && projects.length > 0} + {#if !loading && allProjects.length > 0}
{/each}
- {:else if projects.length === 0} + {:else if activeProjects.length === 0 && deletedProjects.length === 0}
- {:else if filteredProjects.length === 0} + {:else if filteredActive().length === 0 && filteredDeleted().length === 0}
{:else} -
- {#each filteredProjects as project} - - -
-
- {project.name} - {#if project.description} - {project.description} - {/if} + + {#if filteredActive().length > 0} +
+ {#each filteredActive() as project} + + +
+
+ {project.name} + {#if project.description} + {project.description} + {/if} +
-
- - -
-
- Created {formatDate(project.createdAt)} + + +
+
+ Created {formatDate(project.createdAt)} +
+
+ e.stopPropagation()} + > + View Project + + + + + {#snippet child({ props })} + + {/snippet} + + + + Delete Project? + + "{project.name}" will be moved to trash. Historical + logs, traces and metrics remain accessible for 30 days, + after which they are permanently deleted. + + + + Cancel + deleteProject(project.id)} + > + Move to Trash + + + + +
-
- e.stopPropagation()} - > - View Project - - - - - {#snippet child({ props })} - - {/snippet} - - - - Delete Project? - - This action cannot be undone. This will permanently - delete the project "{project.name}" and all associated - logs. - - - - Cancel - deleteProject(project.id)} + + + {/each} +
+ {/if} + + + {#if filteredDeleted().length > 0} +
+

+ Deleted +

+
+ {#each filteredDeleted() as project} + + +
+
+
+ {project.name} + + Deleted + +
+ {#if project.description} + {project.description} - Delete Project - - - - -
-
- - - {/each} -
+ {/if} +
+
+ + +
+
+ Deleted {formatDate(project.deletedAt!)} +
+
+ e.stopPropagation()} + > + View Logs + + +
+
+ + + {/each} +
+
+ {/if} {/if}
diff --git a/packages/frontend/vitest.config.ts b/packages/frontend/vitest.config.ts index 5cd40c34..597e698f 100644 --- a/packages/frontend/vitest.config.ts +++ b/packages/frontend/vitest.config.ts @@ -1,6 +1,10 @@ import { defineConfig } from 'vitest/config'; import { svelte } from '@sveltejs/vite-plugin-svelte'; import { svelteTesting } from '@testing-library/svelte/vite'; +import { fileURLToPath } from 'url'; +import path from 'path'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); export default defineConfig({ plugins: [ @@ -12,11 +16,14 @@ export default defineConfig({ environment: 'jsdom', include: ['src/**/*.test.ts'], setupFiles: ['./src/test-setup.ts'], - alias: { - $lib: new URL('./src/lib', import.meta.url).pathname, - }, }, resolve: { conditions: ['browser'], + alias: { + // Use fileURLToPath so paths with spaces are decoded correctly + // (new URL().pathname encodes spaces as %20 which breaks Vite resolution) + $lib: path.resolve(__dirname, 'src/lib'), + '@logtide/shared': path.resolve(__dirname, '../../packages/shared/src/index.ts'), + }, }, }); diff --git a/packages/reservoir/src/buffered/reservoir-buffered.ts b/packages/reservoir/src/buffered/reservoir-buffered.ts index 89aa66db..69d3ac80 100644 --- a/packages/reservoir/src/buffered/reservoir-buffered.ts +++ b/packages/reservoir/src/buffered/reservoir-buffered.ts @@ -131,6 +131,7 @@ export class ReservoirBuffered implements IReservoir { getMetricLabelValues(...args: Parameters): ReturnType { return this.inner.getMetricLabelValues(...args); } deleteMetricsByTimeRange(...args: Parameters): ReturnType { return this.inner.deleteMetricsByTimeRange(...args); } getMetricsOverview(...args: Parameters): ReturnType { return this.inner.getMetricsOverview(...args); } + purgeProject(...args: Parameters): ReturnType { return this.inner.purgeProject(...args); } getEngineType(): ReturnType { return this.inner.getEngineType(); } getEngine(): StorageEngine { return this.inner.getEngine(); } async close(): Promise { await this.stop(); await this.inner.close(); } diff --git a/packages/reservoir/src/client.test.ts b/packages/reservoir/src/client.test.ts index 42b15889..e4f8789e 100644 --- a/packages/reservoir/src/client.test.ts +++ b/packages/reservoir/src/client.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { Reservoir } from './client.js'; import type { StorageConfig, QueryParams, AggregateParams } from './core/types.js'; @@ -69,4 +69,55 @@ describe('Reservoir', () => { const reservoir = new Reservoir('timescale', config); await expect(reservoir.close()).resolves.toBeUndefined(); }); + + describe('purgeProject', () => { + it('throws if called before initialization', async () => { + const reservoir = new Reservoir('timescale', config); + await expect(reservoir.purgeProject('p1')).rejects.toThrow('Reservoir not initialized'); + }); + + it('delegates to all three engine delete methods and sums deleted counts', async () => { + const reservoir = new Reservoir('timescale', config); + const r = reservoir as any; + r.initialized = true; + r.engine = { + deleteByTimeRange: vi.fn().mockResolvedValue({ deleted: 5 }), + deleteSpansByTimeRange: vi.fn().mockResolvedValue({ deleted: 3 }), + deleteMetricsByTimeRange: vi.fn().mockResolvedValue({ deleted: 2 }), + }; + + const result = await reservoir.purgeProject('test-project-id'); + + expect(result.deleted).toBe(10); + expect(r.engine.deleteByTimeRange).toHaveBeenCalledWith({ + projectId: 'test-project-id', + from: new Date(0), + to: new Date('2100-01-01T00:00:00Z'), + }); + expect(r.engine.deleteSpansByTimeRange).toHaveBeenCalledWith({ + projectId: 'test-project-id', + from: new Date(0), + to: new Date('2100-01-01T00:00:00Z'), + }); + expect(r.engine.deleteMetricsByTimeRange).toHaveBeenCalledWith({ + projectId: 'test-project-id', + from: new Date(0), + to: new Date('2100-01-01T00:00:00Z'), + }); + }); + + it('returns zero deleted when engine returns empty counts', async () => { + const reservoir = new Reservoir('timescale', config); + const r = reservoir as any; + r.initialized = true; + r.engine = { + deleteByTimeRange: vi.fn().mockResolvedValue({ deleted: 0 }), + deleteSpansByTimeRange: vi.fn().mockResolvedValue({ deleted: 0 }), + deleteMetricsByTimeRange: vi.fn().mockResolvedValue({ deleted: 0 }), + }; + + const result = await reservoir.purgeProject('empty-project'); + expect(result.deleted).toBe(0); + }); + }); }); diff --git a/packages/reservoir/src/client.ts b/packages/reservoir/src/client.ts index 0c8e5c02..124932c8 100644 --- a/packages/reservoir/src/client.ts +++ b/packages/reservoir/src/client.ts @@ -277,6 +277,23 @@ export class Reservoir implements IReservoir { return this.engine.getMetricsOverview(params); } + /** + * Hard-purge all logs, spans, and metrics for a project. + * Implements IReservoir.purgeProject by calling the three time-range delete + * methods with the full time span so nothing is left behind. + */ + async purgeProject(projectId: string): Promise { + this.ensureInitialized(); + const from = new Date(0); + const to = new Date('2100-01-01T00:00:00Z'); + const [logs, spans, metrics] = await Promise.all([ + this.engine.deleteByTimeRange({ projectId, from, to }), + this.engine.deleteSpansByTimeRange({ projectId, from, to }), + this.engine.deleteMetricsByTimeRange({ projectId, from, to }), + ]); + return { deleted: logs.deleted + spans.deleted + metrics.deleted }; + } + getEngineType(): EngineType { return this.engine.getCapabilities().engine; } diff --git a/packages/reservoir/src/core/reservoir-interface.ts b/packages/reservoir/src/core/reservoir-interface.ts index a07b830c..2c4ec7c4 100644 --- a/packages/reservoir/src/core/reservoir-interface.ts +++ b/packages/reservoir/src/core/reservoir-interface.ts @@ -104,6 +104,14 @@ export interface IReservoir { deleteMetricsByTimeRange(params: DeleteMetricsByTimeRangeParams): Promise; getMetricsOverview(params: MetricsOverviewParams): Promise; + /** + * Hard-purge all logs, spans, and metrics for a project across every + * storage engine. Called by the background hard-delete worker after the + * soft-delete grace window expires, before the projects row is removed from + * Postgres. + */ + purgeProject(projectId: string): Promise; + // Lifecycle & introspection getEngineType(): EngineType; close(): Promise; diff --git a/packages/reservoir/src/test-global-setup.ts b/packages/reservoir/src/test-global-setup.ts new file mode 100644 index 00000000..14801aae --- /dev/null +++ b/packages/reservoir/src/test-global-setup.ts @@ -0,0 +1,27 @@ +/** + * Vitest global setup: auto-detect infrastructure availability. + * + * Sets SKIP_REDIS_TESTS=1 when a Redis server is not reachable so that + * integration tests degrade gracefully to "skipped" instead of "failed". + */ +import Redis from 'ioredis'; + +const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6380'; + +export async function setup() { + const redis = new Redis(REDIS_URL, { + maxRetriesPerRequest: 0, + lazyConnect: true, + connectTimeout: 1500, + enableReadyCheck: false, + }); + + try { + await redis.connect(); + await redis.ping(); + } catch { + process.env.SKIP_REDIS_TESTS = '1'; + } finally { + redis.disconnect(false); + } +} diff --git a/packages/reservoir/vitest.config.ts b/packages/reservoir/vitest.config.ts index ecaa2a77..6483218a 100644 --- a/packages/reservoir/vitest.config.ts +++ b/packages/reservoir/vitest.config.ts @@ -1,4 +1,8 @@ import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'url'; +import path from 'path'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); export default defineConfig({ test: { @@ -18,5 +22,14 @@ export default defineConfig({ ], }, testTimeout: 10000, + globalSetup: ['./src/test-global-setup.ts'], + }, + resolve: { + alias: { + // @logtide/shared dist/ is not built; alias directly to source so tests + // can resolve the package without a build step. + '@logtide/shared/context': path.resolve(__dirname, '../shared/src/context/index.ts'), + '@logtide/shared': path.resolve(__dirname, '../shared/src/index.ts'), + }, }, }); diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 6202c619..a23e0b36 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -108,6 +108,7 @@ export interface Project { statusPageVisibility: StatusPageVisibility; createdAt: Date; updatedAt: Date; + deletedAt?: Date | null; } // Log types From 2b585763a1a0c4b17d7284c5c15a06b41dd5b1fc Mon Sep 17 00:00:00 2001 From: Polliog Date: Sat, 27 Jun 2026 11:24:19 +0200 Subject: [PATCH 02/23] reject project restore on name or slug conflict --- .../backend/src/modules/projects/routes.ts | 10 +++++++ .../backend/src/modules/projects/service.ts | 16 +++++++++++ .../modules/projects/projects-service.test.ts | 21 ++++++++++++++ .../src/tests/modules/projects/routes.test.ts | 28 +++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/packages/backend/src/modules/projects/routes.ts b/packages/backend/src/modules/projects/routes.ts index e1d09f36..a51672ce 100644 --- a/packages/backend/src/modules/projects/routes.ts +++ b/packages/backend/src/modules/projects/routes.ts @@ -188,6 +188,16 @@ export async function projectsRoutes(fastify: FastifyInstance) { if (error instanceof z.ZodError) { return reply.status(400).send({ error: 'Invalid project ID format' }); } + if (error instanceof Error && error.message.includes('already exists')) { + return reply.status(409).send({ error: error.message }); + } + // Safety net for a concurrent name/slug reuse that slips past the + // pre-check and trips the partial unique index (Postgres 23505). + if ((error as { code?: string })?.code === '23505') { + return reply.status(409).send({ + error: 'A project with this name or slug already exists in this organization', + }); + } throw error; } }); diff --git a/packages/backend/src/modules/projects/service.ts b/packages/backend/src/modules/projects/service.ts index 2b7d0379..d625f001 100644 --- a/packages/backend/src/modules/projects/service.ts +++ b/packages/backend/src/modules/projects/service.ts @@ -452,6 +452,22 @@ export class ProjectsService { return false; } + // Name and slug are only unique among ACTIVE projects (partial unique + // indexes), so while this project sat soft-deleted another active project + // may have taken its name or slug. Restoring would then violate those + // indexes; reject with a clear error instead of surfacing a raw DB failure. + const conflict = await db + .selectFrom('projects') + .select('id') + .where('organization_id', '=', project.organizationId) + .where('deleted_at', 'is', null) + .where((eb) => eb.or([eb('name', '=', project.name), eb('slug', '=', project.slug)])) + .executeTakeFirst(); + + if (conflict) { + throw new Error('A project with this name or slug already exists in this organization'); + } + const result = await db .updateTable('projects') .set({ deleted_at: null }) diff --git a/packages/backend/src/tests/modules/projects/projects-service.test.ts b/packages/backend/src/tests/modules/projects/projects-service.test.ts index 8afc16b7..fe8b9602 100644 --- a/packages/backend/src/tests/modules/projects/projects-service.test.ts +++ b/packages/backend/src/tests/modules/projects/projects-service.test.ts @@ -451,6 +451,27 @@ describe('ProjectsService', () => { const restored = await projectsService.restoreProject(project.id, outsider.id); expect(restored).toBe(false); }); + + it('should throw when an active project has reused the name or slug', async () => { + const { project, user, organization } = await createTestContext(); + + await projectsService.deleteProject(project.id, user.id); + + // A new active project takes the freed name (and slug). + await projectsService.createProject({ + organizationId: organization.id, + userId: user.id, + name: project.name, + }); + + await expect( + projectsService.restoreProject(project.id, user.id) + ).rejects.toThrow('already exists'); + + // The conflicting project stays soft-deleted. + const stillDeleted = await projectsService.getProjectById(project.id, user.id); + expect(stillDeleted?.deletedAt).not.toBeNull(); + }); }); describe('getOrganizationProjectsIncludingDeleted', () => { diff --git a/packages/backend/src/tests/modules/projects/routes.test.ts b/packages/backend/src/tests/modules/projects/routes.test.ts index 17c7d153..66261651 100644 --- a/packages/backend/src/tests/modules/projects/routes.test.ts +++ b/packages/backend/src/tests/modules/projects/routes.test.ts @@ -412,6 +412,34 @@ describe('Projects Routes', () => { expect(body.error).toContain('not deleted'); }); + it('should return 409 when an active project has reused the name', async () => { + // Soft-delete the test project, freeing its name and slug. + await db.updateTable('projects') + .set({ deleted_at: new Date() }) + .where('id', '=', testProject.id) + .execute(); + + // A new active project takes the freed name. + await db.insertInto('projects') + .values({ + organization_id: testOrganization.id, + user_id: testUser.id, + name: testProject.name, + slug: `${testProject.slug}-new`, + }) + .execute(); + + const response = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${testProject.id}/restore`, + headers: { Authorization: `Bearer ${authToken}` }, + }); + + expect(response.statusCode).toBe(409); + const body = JSON.parse(response.payload); + expect(body.error).toContain('already exists'); + }); + it('should return 404 for non-existent project', async () => { const response = await app.inject({ method: 'POST', From cfa33691eb476897252e29550d36e878e855a414 Mon Sep 17 00:00:00 2001 From: Polliog Date: Sat, 27 Jun 2026 11:24:19 +0200 Subject: [PATCH 03/23] let soft-deleted projects be viewed read-only --- CHANGELOG.md | 3 + .../dashboard/projects/[id]/+layout.svelte | 57 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19381def..416153dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **Soft-delete for projects with a 30-day grace window**: deleting a project now moves it to a recoverable "trash" state (`deleted_at`) instead of removing it immediately. Its logs, traces and metrics stay queryable and the project stays viewable read-only during the window (the project detail layout loads with `includeDeleted`, shows a "deleted (read-only)" banner with the permanent-deletion date, and hides the Settings tab); a "Restore" action brings it back. A daily purge worker (3 AM) hard-deletes projects past the grace window, calling `reservoir.purgeProject()` to clear logs/spans/metrics from every storage engine before deleting the row. Name and slug uniqueness moved to partial unique indexes (active projects only) so a deleted project's name/slug can be reused; restoring into a name or slug a new active project has since taken is rejected with a 409 instead of a raw constraint error. Soft-deleted projects are excluded from listings, data-availability widgets and API-key verification (ingestion is refused immediately, with the key-verification cache invalidated on delete). New `POST /api/v1/projects/:id/restore` and `GET /api/v1/projects?includeDeleted=true`. Migrations 050 (soft-delete column + partial indexes) and 051 (drop `ON DELETE CASCADE` from logs/spans/metrics so an out-of-band project delete cannot bulk-wipe data; the purge worker clears reservoir data explicitly first) + ## [1.0.3] - 2026-06-26 A security-focused release. It resolves a batch of privately reported issues (coordinated disclosure via KIberblick.de): cross-tenant read on the dashboard API endpoints, stored XSS via OTLP `service.name` in the service map, an open redirect on the auth-free login/register path, a DNS-rebinding gap in the SSRF guard's HTTP path, a first-admin bootstrap promotion race, and a capability-limit check-then-act race; trace span attributes are now PII-masked as well, and `service.name` is sanitized at ingestion as defense in depth. No database migrations; drop-in upgrade. Alongside the security work: two correctness follow-ups from the multi-engine bug-hunt sweep (issue #255): Sigma detection now honors full SigmaHQ field-modifier chains, and the service-map p95 is a true window percentile on every storage engine. The storage-layer change was validated against real ClickHouse, MongoDB and TimescaleDB. This line also fixes two operational bugs: a Redis memory leak where completed/failed BullMQ jobs were never evicted, and a nightly SigmaHQ sync that re-imported the whole catalog as enabled and auto-created alert rules. Plus a few frontend touch-ups: theme-aware trace/session IDs in the log detail, per-occurrence trace links on the error page, metadata copy buttons, a breadcrumbs timeline and nested metadata columns in log search. diff --git a/packages/frontend/src/routes/dashboard/projects/[id]/+layout.svelte b/packages/frontend/src/routes/dashboard/projects/[id]/+layout.svelte index 931f81a3..781aaabc 100644 --- a/packages/frontend/src/routes/dashboard/projects/[id]/+layout.svelte +++ b/packages/frontend/src/routes/dashboard/projects/[id]/+layout.svelte @@ -8,9 +8,13 @@ import Card from '$lib/components/ui/card/card.svelte'; import CardContent from '$lib/components/ui/card/card-content.svelte'; import Spinner from '$lib/components/Spinner.svelte'; + import Button from '$lib/components/ui/button/button.svelte'; + import RotateCcw from '@lucide/svelte/icons/rotate-ccw'; import * as Tabs from '$lib/components/ui/tabs'; import { layoutStore } from '$lib/stores/layout'; + const HARD_DELETE_GRACE_DAYS = 30; + interface Props { children: import('svelte').Snippet; } @@ -20,6 +24,7 @@ let project = $state(null); let capabilities = $state<{ hasWebVitals: boolean; hasSessions: boolean }>({ hasWebVitals: false, hasSessions: false }); let loading = $state(false); + let restoring = $state(false); let error = $state(''); let lastLoadedKey = $state(null); let maxWidthClass = $state("max-w-7xl"); @@ -65,7 +70,10 @@ tabs.push({ value: 'sessions', label: 'Sessions' }); } tabs.push({ value: 'alerts', label: 'Alerts' }); - tabs.push({ value: 'settings', label: 'Settings' }); + // A soft-deleted project is read-only; its settings cannot be edited. + if (!project?.deletedAt) { + tabs.push({ value: 'settings', label: 'Settings' }); + } return tabs; }); @@ -75,7 +83,9 @@ try { const [response, caps] = await Promise.all([ - projectsAPI.getProjects(orgId), + // includeDeleted so a soft-deleted project stays viewable (read-only) + // during the grace window — its logs/traces/metrics are still around. + projectsAPI.getProjects(orgId, { includeDeleted: true }), projectsAPI.getProjectCapabilities(projId).catch(() => ({ hasWebVitals: false, hasSessions: false })), ]); @@ -117,6 +127,29 @@ const basePath = `/dashboard/projects/${projectId}`; goto(`${basePath}/${tab}`); } + + // Date (en-US, project convention) the project is permanently purged. + const purgeDate = $derived.by(() => { + if (!project?.deletedAt) return null; + const d = new Date(project.deletedAt); + d.setDate(d.getDate() + HARD_DELETE_GRACE_DAYS); + return d.toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' }); + }); + + async function restoreProject() { + if (!$currentOrganization || !projectId) return; + restoring = true; + try { + await projectsAPI.restoreProject($currentOrganization.id, projectId); + toastStore.success('Project restored successfully.'); + lastLoadedKey = null; // force the loader to refetch + await loadProject($currentOrganization.id, projectId); + } catch (e) { + toastStore.error(e instanceof Error ? e.message : 'Failed to restore project'); + } finally { + restoring = false; + } + }
@@ -127,6 +160,26 @@ {:else if project} + {#if project.deletedAt} +
+
+

This project is deleted (read-only)

+

+ Its logs, traces and metrics stay available + {#if purgeDate}until they are permanently deleted on {purgeDate}{:else}for 30 days{/if}. + Restore the project to make it active again. +

+
+ +
+ {/if}

{project.name}

{#if project.description} From f18f78b636653a435f09bf9cb639f0ec1a87dd10 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:07:08 +0200 Subject: [PATCH 04/23] add receivers tables and schema types --- packages/backend/migrations/052_receivers.sql | 30 +++++++++++++++ .../backend/src/database/tenant-tables.ts | 3 +- packages/backend/src/database/types.ts | 37 +++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 packages/backend/migrations/052_receivers.sql diff --git a/packages/backend/migrations/052_receivers.sql b/packages/backend/migrations/052_receivers.sql new file mode 100644 index 00000000..812568fc --- /dev/null +++ b/packages/backend/migrations/052_receivers.sql @@ -0,0 +1,30 @@ +-- Inbound webhook receivers (#155): external systems POST events that get +-- normalized into log entries by per-receiver adapters. +CREATE TABLE IF NOT EXISTS receivers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + name TEXT NOT NULL, + adapter_type TEXT NOT NULL CHECK (adapter_type IN ('github', 'uptime', 'generic')), + token_hash TEXT NOT NULL UNIQUE, + field_mapping JSONB, + enabled BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_received_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_receivers_project ON receivers(project_id); + +-- Recent raw/normalized events per receiver, capped at 100 rows per receiver +-- by the worker (pruneEvents). Powers the "recent events" UI. +CREATE TABLE IF NOT EXISTS receiver_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + receiver_id UUID NOT NULL REFERENCES receivers(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('pending', 'processed', 'skipped', 'failed')), + raw_payload JSONB NOT NULL, + normalized JSONB, + error TEXT, + received_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_receiver_events_receiver + ON receiver_events(receiver_id, received_at DESC); diff --git a/packages/backend/src/database/tenant-tables.ts b/packages/backend/src/database/tenant-tables.ts index ff155d88..d91d303f 100644 --- a/packages/backend/src/database/tenant-tables.ts +++ b/packages/backend/src/database/tenant-tables.ts @@ -14,6 +14,7 @@ export const TENANT_TABLES = new Set([ 'pii_masking_rules', 'organization_pii_salts', 'audit_log', 'metrics_hourly_stats', 'metrics_daily_stats', 'metrics', 'metric_exemplars', 'custom_dashboards', 'log_pipelines', 'digest_configs', 'digest_recipients', 'projects', + 'receivers', ]); /** @@ -23,7 +24,7 @@ export const CHILD_TABLES = new Set([ 'monitor_status', 'status_incident_updates', 'incident_alerts', 'incident_comments', 'incident_history', 'stack_frames', 'alert_rule_channels', 'sigma_rule_channels', 'monitor_channels', - 'incident_channels', 'error_group_channels', + 'incident_channels', 'error_group_channels', 'receiver_events', ]); /** Intentionally global tables (no tenant scope). */ diff --git a/packages/backend/src/database/types.ts b/packages/backend/src/database/types.ts index 43d4a90e..72c5e03a 100644 --- a/packages/backend/src/database/types.ts +++ b/packages/backend/src/database/types.ts @@ -924,6 +924,40 @@ export interface WebhookDeliveryAttemptsTable { created_at: Generated; } +// ============================================================================ +// INBOUND WEBHOOK RECEIVERS (#155) +// ============================================================================ + +export interface ReceiversTable { + id: Generated; + project_id: string; + name: string; + adapter_type: string; // 'github' | 'uptime' | 'generic' + token_hash: string; + field_mapping: ColumnType< + Record | null, + Record | null, + Record | null + >; + enabled: Generated; + created_at: Generated; + last_received_at: ColumnType; +} + +export interface ReceiverEventsTable { + id: Generated; + receiver_id: string; + status: string; // 'pending' | 'processed' | 'skipped' | 'failed' + raw_payload: ColumnType< + Record, + Record, + Record + >; + normalized: ColumnType; + error: string | null; + received_at: Generated; +} + // ============================================================================ // PII MASKING TABLES // ============================================================================ @@ -1209,4 +1243,7 @@ export interface Database { // Outbound webhook delivery (#218) webhook_deliveries: WebhookDeliveriesTable; webhook_delivery_attempts: WebhookDeliveryAttemptsTable; + // Inbound webhook receivers (#155) + receivers: ReceiversTable; + receiver_events: ReceiverEventsTable; } From dc83d735674c0b14d9b605e533ef44c1180a395a Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:08:13 +0200 Subject: [PATCH 05/23] add receiver adapter constants and mapping schema --- packages/shared/src/constants/index.ts | 1 + .../src/constants/receiver-constants.ts | 2 ++ packages/shared/src/schemas/index.ts | 1 + packages/shared/src/schemas/receiver.test.ts | 33 +++++++++++++++++++ packages/shared/src/schemas/receiver.ts | 29 ++++++++++++++++ 5 files changed, 66 insertions(+) create mode 100644 packages/shared/src/constants/receiver-constants.ts create mode 100644 packages/shared/src/schemas/receiver.test.ts create mode 100644 packages/shared/src/schemas/receiver.ts diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index 69883500..f6e26b46 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -7,3 +7,4 @@ export * from './siem-constants.js'; export * from './exception-constants.js'; export * from './mitre-constants.js'; export * from './monitoring-constants.js'; +export * from './receiver-constants.js'; diff --git a/packages/shared/src/constants/receiver-constants.ts b/packages/shared/src/constants/receiver-constants.ts new file mode 100644 index 00000000..2fa66b6e --- /dev/null +++ b/packages/shared/src/constants/receiver-constants.ts @@ -0,0 +1,2 @@ +export const RECEIVER_ADAPTER_TYPES = ['github', 'uptime', 'generic'] as const; +export type ReceiverAdapterType = (typeof RECEIVER_ADAPTER_TYPES)[number]; diff --git a/packages/shared/src/schemas/index.ts b/packages/shared/src/schemas/index.ts index 9edd8f51..56d849ab 100644 --- a/packages/shared/src/schemas/index.ts +++ b/packages/shared/src/schemas/index.ts @@ -71,3 +71,4 @@ export type AlertRuleInput = z.infer; export * from './metadata-filter.js'; export * from './webhook-events.js'; +export * from './receiver.js'; diff --git a/packages/shared/src/schemas/receiver.test.ts b/packages/shared/src/schemas/receiver.test.ts new file mode 100644 index 00000000..aeeaad07 --- /dev/null +++ b/packages/shared/src/schemas/receiver.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from 'vitest'; +import { receiverFieldMappingSchema, RECEIVER_ADAPTER_TYPES } from '../index.js'; + +describe('receiverFieldMappingSchema', () => { + it('accepts a full valid mapping', () => { + const result = receiverFieldMappingSchema.safeParse({ + message: 'msg.text', + level: 'severity', + service: 'source.app', + timestamp: 'ts', + levelMap: { crit: 'critical', warning: 'warn' }, + defaults: { level: 'info', service: 'external' }, + }); + expect(result.success).toBe(true); + }); + + it('accepts an empty mapping', () => { + expect(receiverFieldMappingSchema.safeParse({}).success).toBe(true); + }); + + it('rejects unknown keys', () => { + expect(receiverFieldMappingSchema.safeParse({ nope: 'x' }).success).toBe(false); + }); + + it('rejects invalid levels in levelMap and defaults', () => { + expect(receiverFieldMappingSchema.safeParse({ levelMap: { a: 'verbose' } }).success).toBe(false); + expect(receiverFieldMappingSchema.safeParse({ defaults: { level: 'verbose' } }).success).toBe(false); + }); + + it('exposes the adapter type list', () => { + expect(RECEIVER_ADAPTER_TYPES).toEqual(['github', 'uptime', 'generic']); + }); +}); diff --git a/packages/shared/src/schemas/receiver.ts b/packages/shared/src/schemas/receiver.ts new file mode 100644 index 00000000..4702cecc --- /dev/null +++ b/packages/shared/src/schemas/receiver.ts @@ -0,0 +1,29 @@ +import { z } from 'zod'; +import { LOG_LEVELS, RECEIVER_ADAPTER_TYPES } from '../constants/index.js'; + +export const receiverAdapterTypeSchema = z.enum(RECEIVER_ADAPTER_TYPES); + +const dotPath = z.string().min(1).max(200); + +/** + * Field mapping for the 'generic' receiver adapter (#155). + * Values are dot-paths into the raw payload (e.g. "error.message"). + */ +export const receiverFieldMappingSchema = z + .object({ + message: dotPath.optional(), + level: dotPath.optional(), + service: dotPath.optional(), + timestamp: dotPath.optional(), + levelMap: z.record(z.enum(LOG_LEVELS)).optional(), + defaults: z + .object({ + level: z.enum(LOG_LEVELS).optional(), + service: z.string().min(1).max(100).optional(), + }) + .strict() + .optional(), + }) + .strict(); + +export type ReceiverFieldMapping = z.infer; From e7791ba9738d087564023b99d01be80488b8ad86 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:10:19 +0200 Subject: [PATCH 06/23] add receiver adapter core and generic adapter --- .../src/modules/receivers/adapters/generic.ts | 84 +++++++++++++++++++ .../src/modules/receivers/adapters/types.ts | 22 +++++ .../modules/receivers/generic-adapter.test.ts | 84 +++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 packages/backend/src/modules/receivers/adapters/generic.ts create mode 100644 packages/backend/src/modules/receivers/adapters/types.ts create mode 100644 packages/backend/src/tests/modules/receivers/generic-adapter.test.ts diff --git a/packages/backend/src/modules/receivers/adapters/generic.ts b/packages/backend/src/modules/receivers/adapters/generic.ts new file mode 100644 index 00000000..09079726 --- /dev/null +++ b/packages/backend/src/modules/receivers/adapters/generic.ts @@ -0,0 +1,84 @@ +import type { LogLevel } from '@logtide/shared'; +import { LOG_LEVELS } from '@logtide/shared'; +import type { ReceiverAdapter } from './types.js'; + +/** Resolve a dot-path ("a.b.c") into a nested object. */ +export function getPath(obj: unknown, path: string): unknown { + let current: unknown = obj; + for (const key of path.split('.')) { + if (current === null || typeof current !== 'object') return undefined; + current = (current as Record)[key]; + } + return current; +} + +const LEVEL_SYNONYMS: Record = { + warning: 'warn', + fatal: 'critical', + err: 'error', + crit: 'critical', +}; + +function coerceLevel( + value: unknown, + levelMap: Record | undefined, + fallback: LogLevel +): LogLevel { + if (typeof value !== 'string' && typeof value !== 'number') return fallback; + const raw = String(value).toLowerCase(); + if (levelMap) { + for (const [key, mapped] of Object.entries(levelMap)) { + if (key.toLowerCase() === raw) return mapped; + } + } + if ((LOG_LEVELS as readonly string[]).includes(raw)) return raw as LogLevel; + if (raw in LEVEL_SYNONYMS) return LEVEL_SYNONYMS[raw]; + return fallback; +} + +export const genericAdapter: ReceiverAdapter = (payload, receiver) => { + const mapping = receiver.fieldMapping ?? {}; + const defaults = mapping.defaults ?? {}; + + const messageRaw = mapping.message ? getPath(payload, mapping.message) : undefined; + const message = + typeof messageRaw === 'string' && messageRaw.length > 0 ? messageRaw : 'Received event'; + + const serviceRaw = mapping.service ? getPath(payload, mapping.service) : undefined; + const service = ( + typeof serviceRaw === 'string' && serviceRaw.length > 0 + ? serviceRaw + : (defaults.service ?? receiver.name) + ).slice(0, 100); + + const level = coerceLevel( + mapping.level ? getPath(payload, mapping.level) : undefined, + mapping.levelMap, + defaults.level ?? 'info' + ); + + let time = new Date().toISOString(); + if (mapping.timestamp) { + const ts = getPath(payload, mapping.timestamp); + if (typeof ts === 'string' || typeof ts === 'number') { + const parsed = new Date(ts); + if (!Number.isNaN(parsed.getTime())) time = parsed.toISOString(); + } + } + + return { + kind: 'logs', + logs: [ + { + time, + service, + level, + message, + metadata: { + receiver: { id: receiver.id, name: receiver.name, adapter: 'generic' }, + payload, + }, + }, + ], + }; +}; diff --git a/packages/backend/src/modules/receivers/adapters/types.ts b/packages/backend/src/modules/receivers/adapters/types.ts new file mode 100644 index 00000000..67c5dd04 --- /dev/null +++ b/packages/backend/src/modules/receivers/adapters/types.ts @@ -0,0 +1,22 @@ +import type { LogInput, ReceiverAdapterType, ReceiverFieldMapping } from '@logtide/shared'; + +export interface AdapterReceiverInfo { + id: string; + name: string; + adapterType: ReceiverAdapterType; + fieldMapping: ReceiverFieldMapping | null; +} + +export type AdapterResult = + | { kind: 'logs'; logs: LogInput[] } + | { kind: 'skipped'; reason: string }; + +/** + * A receiver adapter is a pure function turning a raw external payload into + * zero or more log entries. New adapters plug in via adapters/index.ts and + * need no changes to the receiver core. + */ +export type ReceiverAdapter = ( + payload: Record, + receiver: AdapterReceiverInfo +) => AdapterResult; diff --git a/packages/backend/src/tests/modules/receivers/generic-adapter.test.ts b/packages/backend/src/tests/modules/receivers/generic-adapter.test.ts new file mode 100644 index 00000000..d89a8081 --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/generic-adapter.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect } from 'vitest'; +import { genericAdapter, getPath } from '../../../modules/receivers/adapters/generic.js'; +import type { AdapterReceiverInfo } from '../../../modules/receivers/adapters/types.js'; + +function receiver(fieldMapping: AdapterReceiverInfo['fieldMapping'] = null): AdapterReceiverInfo { + return { id: 'r-1', name: 'my receiver', adapterType: 'generic', fieldMapping }; +} + +describe('getPath', () => { + it('resolves nested dot paths', () => { + expect(getPath({ a: { b: { c: 42 } } }, 'a.b.c')).toBe(42); + }); + it('returns undefined for missing segments and non-objects', () => { + expect(getPath({ a: 1 }, 'a.b')).toBeUndefined(); + expect(getPath(null, 'a')).toBeUndefined(); + }); +}); + +describe('genericAdapter', () => { + it('maps fields via dot paths', () => { + const result = genericAdapter( + { sev: 'ERROR', txt: 'disk full', app: 'billing', ts: '2026-07-01T10:00:00Z' }, + receiver({ message: 'txt', level: 'sev', service: 'app', timestamp: 'ts' }) + ); + expect(result.kind).toBe('logs'); + if (result.kind !== 'logs') return; + expect(result.logs).toHaveLength(1); + const log = result.logs[0]; + expect(log.message).toBe('disk full'); + expect(log.level).toBe('error'); + expect(log.service).toBe('billing'); + expect(log.time).toBe('2026-07-01T10:00:00.000Z'); + expect((log.metadata as any).payload.txt).toBe('disk full'); + }); + + it('applies levelMap before builtin coercion', () => { + const result = genericAdapter( + { sev: 'CRIT', txt: 'x' }, + receiver({ message: 'txt', level: 'sev', levelMap: { crit: 'critical' } }) + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect(result.logs[0].level).toBe('critical'); + }); + + it('coerces common synonyms (warning, fatal)', () => { + const r1 = genericAdapter({ l: 'WARNING' }, receiver({ level: 'l' })); + const r2 = genericAdapter({ l: 'fatal' }, receiver({ level: 'l' })); + if (r1.kind !== 'logs' || r2.kind !== 'logs') throw new Error('expected logs'); + expect(r1.logs[0].level).toBe('warn'); + expect(r2.logs[0].level).toBe('critical'); + }); + + it('falls back to defaults and receiver name without mapping', () => { + const result = genericAdapter({ anything: true }, receiver(null)); + if (result.kind !== 'logs') throw new Error('expected logs'); + const log = result.logs[0]; + expect(log.message).toBe('Received event'); + expect(log.level).toBe('info'); + expect(log.service).toBe('my receiver'); + }); + + it('uses configured defaults over builtin fallbacks', () => { + const result = genericAdapter( + {}, + receiver({ defaults: { level: 'warn', service: 'ext' } }) + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect(result.logs[0].level).toBe('warn'); + expect(result.logs[0].service).toBe('ext'); + }); + + it('ignores unparseable timestamps', () => { + const result = genericAdapter({ ts: 'not-a-date' }, receiver({ timestamp: 'ts' })); + if (result.kind !== 'logs') throw new Error('expected logs'); + // falls back to "now": still a valid ISO string + expect(new Date(result.logs[0].time as string).getTime()).not.toBeNaN(); + }); + + it('truncates service to 100 chars', () => { + const result = genericAdapter({ s: 'x'.repeat(200) }, receiver({ service: 's' })); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect((result.logs[0].service as string).length).toBe(100); + }); +}); From 82cc9d2f21a4e0c1ae6ed528fcf16ccb3beb9ccd Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:10:20 +0200 Subject: [PATCH 07/23] add github receiver adapter --- .../src/modules/receivers/adapters/github.ts | 111 ++++++++++++++++++ .../modules/receivers/github-adapter.test.ts | 100 ++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 packages/backend/src/modules/receivers/adapters/github.ts create mode 100644 packages/backend/src/tests/modules/receivers/github-adapter.test.ts diff --git a/packages/backend/src/modules/receivers/adapters/github.ts b/packages/backend/src/modules/receivers/adapters/github.ts new file mode 100644 index 00000000..80a38a45 --- /dev/null +++ b/packages/backend/src/modules/receivers/adapters/github.ts @@ -0,0 +1,111 @@ +import type { LogInput, LogLevel } from '@logtide/shared'; +import type { ReceiverAdapter } from './types.js'; + +function conclusionLevel(conclusion: string | null): LogLevel { + switch (conclusion) { + case 'success': + return 'info'; + case 'cancelled': + case 'skipped': + case 'neutral': + return 'warn'; + case 'failure': + case 'timed_out': + case 'startup_failure': + case 'action_required': + return 'error'; + default: + return 'info'; + } +} + +function isoOrNow(value: unknown): string { + if (typeof value === 'string') { + const parsed = new Date(value); + if (!Number.isNaN(parsed.getTime())) return parsed.toISOString(); + } + return new Date().toISOString(); +} + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' ? (value as Record) : undefined; +} + +/** + * GitHub webhook adapter (#155). Detects events by payload shape rather than + * the X-GitHub-Event header so replayed/stored payloads normalize identically. + * Supported: workflow_run (completed), deployment_status. Ping and everything + * else is skipped. + */ +export const githubAdapter: ReceiverAdapter = (payload, receiver) => { + if ('zen' in payload && 'hook_id' in payload) { + return { kind: 'skipped', reason: 'github ping event' }; + } + + const repoName = asRecord(payload.repository)?.full_name; + const service = (typeof repoName === 'string' ? repoName : 'github').slice(0, 100); + const senderLogin = asRecord(payload.sender)?.login; + const actor = typeof senderLogin === 'string' ? senderLogin : undefined; + const action = typeof payload.action === 'string' ? payload.action : undefined; + + const run = asRecord(payload.workflow_run); + if (run) { + if (action !== 'completed') { + return { kind: 'skipped', reason: `workflow_run action "${action ?? 'unknown'}" ignored` }; + } + const conclusion = typeof run.conclusion === 'string' ? run.conclusion : null; + const workflowName = typeof run.name === 'string' ? run.name : 'workflow'; + const log: LogInput = { + time: isoOrNow(run.updated_at), + service, + level: conclusionLevel(conclusion), + message: `Workflow ${workflowName} completed: ${conclusion ?? 'unknown'}`, + metadata: { + receiver: { id: receiver.id, name: receiver.name, adapter: 'github' }, + event: 'workflow_run', + action, + repository: service, + workflow: workflowName, + conclusion, + run_id: run.id, + run_url: run.html_url, + branch: run.head_branch, + actor, + }, + }; + return { kind: 'logs', logs: [log] }; + } + + const status = asRecord(payload.deployment_status); + if (status) { + const deployment = asRecord(payload.deployment); + const state = typeof status.state === 'string' ? status.state : 'unknown'; + const environment = + typeof status.environment === 'string' + ? status.environment + : typeof deployment?.environment === 'string' + ? deployment.environment + : 'unknown'; + const level: LogLevel = + state === 'success' ? 'info' : state === 'failure' || state === 'error' ? 'error' : 'info'; + const log: LogInput = { + time: isoOrNow(status.updated_at), + service, + level, + message: `Deployment to ${environment}: ${state}`, + metadata: { + receiver: { id: receiver.id, name: receiver.name, adapter: 'github' }, + event: 'deployment_status', + action, + repository: service, + environment, + state, + deployment_url: status.target_url, + actor, + }, + }; + return { kind: 'logs', logs: [log] }; + } + + return { kind: 'skipped', reason: 'unsupported github event' }; +}; diff --git a/packages/backend/src/tests/modules/receivers/github-adapter.test.ts b/packages/backend/src/tests/modules/receivers/github-adapter.test.ts new file mode 100644 index 00000000..c34ce8c8 --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/github-adapter.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from 'vitest'; +import { githubAdapter } from '../../../modules/receivers/adapters/github.js'; +import type { AdapterReceiverInfo } from '../../../modules/receivers/adapters/types.js'; + +const receiver: AdapterReceiverInfo = { + id: 'r-1', + name: 'gh', + adapterType: 'github', + fieldMapping: null, +}; + +describe('githubAdapter', () => { + it('skips ping events', () => { + const result = githubAdapter({ zen: 'Keep it simple.', hook_id: 1 }, receiver); + expect(result.kind).toBe('skipped'); + }); + + it('maps a failed workflow_run to an error log', () => { + const result = githubAdapter( + { + action: 'completed', + workflow_run: { + id: 42, + name: 'CI', + conclusion: 'failure', + html_url: 'https://github.com/acme/app/actions/runs/42', + head_branch: 'main', + updated_at: '2026-07-01T10:00:00Z', + }, + repository: { full_name: 'acme/app' }, + sender: { login: 'octocat' }, + }, + receiver + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + const log = result.logs[0]; + expect(log.level).toBe('error'); + expect(log.service).toBe('acme/app'); + expect(log.message).toBe('Workflow CI completed: failure'); + const meta = log.metadata as any; + expect(meta.event).toBe('workflow_run'); + expect(meta.run_id).toBe(42); + expect(meta.branch).toBe('main'); + expect(meta.actor).toBe('octocat'); + }); + + it('maps a successful workflow_run to info and cancelled to warn', () => { + const base = { + action: 'completed', + repository: { full_name: 'acme/app' }, + }; + const ok = githubAdapter( + { ...base, workflow_run: { name: 'CI', conclusion: 'success', updated_at: '2026-07-01T10:00:00Z' } }, + receiver + ); + const cancelled = githubAdapter( + { ...base, workflow_run: { name: 'CI', conclusion: 'cancelled', updated_at: '2026-07-01T10:00:00Z' } }, + receiver + ); + if (ok.kind !== 'logs' || cancelled.kind !== 'logs') throw new Error('expected logs'); + expect(ok.logs[0].level).toBe('info'); + expect(cancelled.logs[0].level).toBe('warn'); + }); + + it('skips non-completed workflow_run actions', () => { + const result = githubAdapter( + { action: 'requested', workflow_run: { name: 'CI' }, repository: { full_name: 'acme/app' } }, + receiver + ); + expect(result.kind).toBe('skipped'); + }); + + it('maps deployment_status events', () => { + const result = githubAdapter( + { + action: 'created', + deployment_status: { + state: 'failure', + environment: 'production', + updated_at: '2026-07-01T10:00:00Z', + }, + deployment: { environment: 'production' }, + repository: { full_name: 'acme/app' }, + sender: { login: 'octocat' }, + }, + receiver + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect(result.logs[0].level).toBe('error'); + expect(result.logs[0].message).toBe('Deployment to production: failure'); + }); + + it('skips unsupported events', () => { + const result = githubAdapter( + { action: 'opened', issue: { number: 1 }, repository: { full_name: 'acme/app' } }, + receiver + ); + expect(result.kind).toBe('skipped'); + }); +}); From bb2c71f32fae4816a2bd05734cfa403dba554ecf Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:10:20 +0200 Subject: [PATCH 08/23] add uptime receiver adapter --- .../src/modules/receivers/adapters/index.ts | 19 ++++ .../src/modules/receivers/adapters/uptime.ts | 94 +++++++++++++++++++ .../modules/receivers/uptime-adapter.test.ts | 93 ++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 packages/backend/src/modules/receivers/adapters/index.ts create mode 100644 packages/backend/src/modules/receivers/adapters/uptime.ts create mode 100644 packages/backend/src/tests/modules/receivers/uptime-adapter.test.ts diff --git a/packages/backend/src/modules/receivers/adapters/index.ts b/packages/backend/src/modules/receivers/adapters/index.ts new file mode 100644 index 00000000..e946d364 --- /dev/null +++ b/packages/backend/src/modules/receivers/adapters/index.ts @@ -0,0 +1,19 @@ +import type { ReceiverAdapterType } from '@logtide/shared'; +import type { ReceiverAdapter } from './types.js'; +import { genericAdapter } from './generic.js'; +import { githubAdapter } from './github.js'; +import { uptimeAdapter } from './uptime.js'; + +const ADAPTERS: Record = { + generic: genericAdapter, + github: githubAdapter, + uptime: uptimeAdapter, +}; + +export function getAdapter(type: ReceiverAdapterType): ReceiverAdapter { + const adapter = ADAPTERS[type]; + if (!adapter) throw new Error(`No adapter registered for type "${type}"`); + return adapter; +} + +export type { AdapterReceiverInfo, AdapterResult, ReceiverAdapter } from './types.js'; diff --git a/packages/backend/src/modules/receivers/adapters/uptime.ts b/packages/backend/src/modules/receivers/adapters/uptime.ts new file mode 100644 index 00000000..ddfdd258 --- /dev/null +++ b/packages/backend/src/modules/receivers/adapters/uptime.ts @@ -0,0 +1,94 @@ +import type { LogInput, LogLevel } from '@logtide/shared'; +import type { ReceiverAdapter } from './types.js'; + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' ? (value as Record) : undefined; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +/** + * Uptime monitoring adapter (#155). Shape-detects two providers: + * - Uptime Robot: flat payload with alertType (1=down, 2=up, 3=ssl expiry) + * - Better Stack: { data: { type: 'incident', attributes: {...} } } + */ +export const uptimeAdapter: ReceiverAdapter = (payload, receiver) => { + // Uptime Robot + if ('alertType' in payload || 'monitorFriendlyName' in payload) { + const alertType = Number(payload.alertType); + const monitor = str(payload.monitorFriendlyName) ?? str(payload.monitorURL) ?? 'monitor'; + const details = str(payload.alertDetails); + let level: LogLevel; + let message: string; + if (alertType === 1) { + level = 'error'; + message = `Monitor ${monitor} is DOWN${details ? `: ${details}` : ''}`; + } else if (alertType === 2) { + level = 'info'; + message = `Monitor ${monitor} is UP${details ? `: ${details}` : ''}`; + } else if (alertType === 3) { + level = 'warn'; + message = `Monitor ${monitor} SSL certificate expiry${details ? `: ${details}` : ''}`; + } else { + return { kind: 'skipped', reason: `unknown uptime robot alertType "${String(payload.alertType)}"` }; + } + const log: LogInput = { + time: new Date().toISOString(), + service: monitor.slice(0, 100), + level, + message, + metadata: { + receiver: { id: receiver.id, name: receiver.name, adapter: 'uptime' }, + provider: 'uptimerobot', + monitor_id: payload.monitorID, + monitor_url: payload.monitorURL, + alert_type: str(payload.alertTypeFriendlyName) ?? alertType, + details, + }, + }; + return { kind: 'logs', logs: [log] }; + } + + // Better Stack + const attributes = asRecord(asRecord(payload.data)?.attributes); + if (attributes) { + const status = (str(attributes.status) ?? '').toLowerCase(); + const monitor = str(attributes.name) ?? str(attributes.url) ?? 'monitor'; + const cause = str(attributes.cause) ?? 'unknown cause'; + let level: LogLevel; + let message: string; + if (status === 'started') { + level = 'error'; + message = `Incident started: ${cause}`; + } else if (status === 'resolved') { + level = 'info'; + message = `Incident resolved: ${cause}`; + } else if (status === 'acknowledged') { + level = 'info'; + message = `Incident acknowledged: ${cause}`; + } else { + return { kind: 'skipped', reason: `unknown better stack incident status "${status}"` }; + } + const log: LogInput = { + time: new Date().toISOString(), + service: monitor.slice(0, 100), + level, + message, + metadata: { + receiver: { id: receiver.id, name: receiver.name, adapter: 'uptime' }, + provider: 'betterstack', + incident_id: asRecord(payload.data)?.id, + status, + cause, + monitor_url: attributes.url, + started_at: attributes.started_at, + resolved_at: attributes.resolved_at, + }, + }; + return { kind: 'logs', logs: [log] }; + } + + return { kind: 'skipped', reason: 'unrecognized uptime payload shape' }; +}; diff --git a/packages/backend/src/tests/modules/receivers/uptime-adapter.test.ts b/packages/backend/src/tests/modules/receivers/uptime-adapter.test.ts new file mode 100644 index 00000000..75c8de16 --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/uptime-adapter.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect } from 'vitest'; +import { uptimeAdapter } from '../../../modules/receivers/adapters/uptime.js'; +import type { AdapterReceiverInfo } from '../../../modules/receivers/adapters/types.js'; + +const receiver: AdapterReceiverInfo = { + id: 'r-1', + name: 'uptime', + adapterType: 'uptime', + fieldMapping: null, +}; + +describe('uptimeAdapter - Uptime Robot', () => { + it('maps a down alert (alertType 1) to error', () => { + const result = uptimeAdapter( + { + monitorID: '777', + monitorFriendlyName: 'API prod', + monitorURL: 'https://api.example.com', + alertType: '1', + alertTypeFriendlyName: 'Down', + alertDetails: 'Connection Timeout', + }, + receiver + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + const log = result.logs[0]; + expect(log.level).toBe('error'); + expect(log.service).toBe('API prod'); + expect(log.message).toBe('Monitor API prod is DOWN: Connection Timeout'); + expect((log.metadata as any).monitor_url).toBe('https://api.example.com'); + }); + + it('maps an up alert (alertType 2) to info and ssl expiry (3) to warn', () => { + const up = uptimeAdapter( + { monitorFriendlyName: 'API prod', alertType: 2, alertTypeFriendlyName: 'Up' }, + receiver + ); + const ssl = uptimeAdapter( + { monitorFriendlyName: 'API prod', alertType: 3, alertTypeFriendlyName: 'SSL expiry' }, + receiver + ); + if (up.kind !== 'logs' || ssl.kind !== 'logs') throw new Error('expected logs'); + expect(up.logs[0].level).toBe('info'); + expect(up.logs[0].message).toBe('Monitor API prod is UP'); + expect(ssl.logs[0].level).toBe('warn'); + }); +}); + +describe('uptimeAdapter - Better Stack', () => { + it('maps a started incident to error', () => { + const result = uptimeAdapter( + { + data: { + id: 'inc-1', + type: 'incident', + attributes: { + name: 'API prod', + cause: 'Status 500', + status: 'Started', + started_at: '2026-07-01T10:00:00Z', + url: 'https://api.example.com', + }, + }, + }, + receiver + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect(result.logs[0].level).toBe('error'); + expect(result.logs[0].service).toBe('API prod'); + expect(result.logs[0].message).toBe('Incident started: Status 500'); + }); + + it('maps a resolved incident to info', () => { + const result = uptimeAdapter( + { + data: { + type: 'incident', + attributes: { name: 'API prod', cause: 'Status 500', status: 'Resolved' }, + }, + }, + receiver + ); + if (result.kind !== 'logs') throw new Error('expected logs'); + expect(result.logs[0].level).toBe('info'); + expect(result.logs[0].message).toBe('Incident resolved: Status 500'); + }); +}); + +describe('uptimeAdapter - unknown shapes', () => { + it('skips unrecognized payloads', () => { + expect(uptimeAdapter({ hello: 'world' }, receiver).kind).toBe('skipped'); + }); +}); From 2eee27347f47e2e14fa5bbc51c25cb769d9b3aae Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:11:52 +0200 Subject: [PATCH 09/23] add receivers service --- .../backend/src/modules/receivers/service.ts | 295 ++++++++++++++++++ .../tests/modules/receivers/service.test.ts | 120 +++++++ 2 files changed, 415 insertions(+) create mode 100644 packages/backend/src/modules/receivers/service.ts create mode 100644 packages/backend/src/tests/modules/receivers/service.test.ts diff --git a/packages/backend/src/modules/receivers/service.ts b/packages/backend/src/modules/receivers/service.ts new file mode 100644 index 00000000..a46bcc14 --- /dev/null +++ b/packages/backend/src/modules/receivers/service.ts @@ -0,0 +1,295 @@ +import crypto from 'crypto'; +import type { LogInput, ReceiverAdapterType, ReceiverFieldMapping } from '@logtide/shared'; +import { db } from '../../database/connection.js'; + +export interface Receiver { + id: string; + projectId: string; + name: string; + adapterType: ReceiverAdapterType; + fieldMapping: ReceiverFieldMapping | null; + enabled: boolean; + createdAt: Date; + lastReceivedAt: Date | null; +} + +export interface IngestReceiver { + id: string; + projectId: string; + organizationId: string; + name: string; + adapterType: ReceiverAdapterType; + fieldMapping: ReceiverFieldMapping | null; + enabled: boolean; + tokenHash: string; +} + +export type ReceiverEventStatus = 'pending' | 'processed' | 'skipped' | 'failed'; + +export interface ReceiverEvent { + id: string; + receiverId: string; + status: ReceiverEventStatus; + rawPayload: Record; + normalized: unknown | null; + error: string | null; + receivedAt: Date; +} + +export interface EventWithReceiver extends IngestReceiver { + eventId: string; + status: ReceiverEventStatus; + rawPayload: Record; +} + +export interface CreateReceiverInput { + projectId: string; + name: string; + adapterType: ReceiverAdapterType; + fieldMapping?: ReceiverFieldMapping | null; +} + +export class ReceiversService { + static readonly MAX_EVENTS_PER_RECEIVER = 100; + + private hashToken(token: string): string { + return crypto.createHash('sha256').update(token).digest('hex'); + } + + generateToken(): string { + return `lr_${crypto.randomBytes(32).toString('hex')}`; + } + + /** Timing-safe comparison of a presented token against the stored hash. */ + tokenMatches(token: string, tokenHash: string): boolean { + const presented = Buffer.from(this.hashToken(token), 'hex'); + const stored = Buffer.from(tokenHash, 'hex'); + return presented.length === stored.length && crypto.timingSafeEqual(presented, stored); + } + + async createReceiver(input: CreateReceiverInput): Promise<{ id: string; token: string }> { + const token = this.generateToken(); + const result = await db + .insertInto('receivers') + .values({ + project_id: input.projectId, + name: input.name, + adapter_type: input.adapterType, + token_hash: this.hashToken(token), + field_mapping: input.fieldMapping ?? null, + }) + .returning(['id']) + .executeTakeFirstOrThrow(); + return { id: result.id, token }; + } + + async listReceivers(projectId: string): Promise { + const rows = await db + .selectFrom('receivers') + .select([ + 'id', + 'project_id', + 'name', + 'adapter_type', + 'field_mapping', + 'enabled', + 'created_at', + 'last_received_at', + ]) + .where('project_id', '=', projectId) + .orderBy('created_at', 'desc') + .execute(); + return rows.map((r) => ({ + id: r.id, + projectId: r.project_id, + name: r.name, + adapterType: r.adapter_type as ReceiverAdapterType, + fieldMapping: (r.field_mapping as ReceiverFieldMapping | null) ?? null, + enabled: r.enabled, + createdAt: new Date(r.created_at), + lastReceivedAt: r.last_received_at ? new Date(r.last_received_at) : null, + })); + } + + async updateReceiver( + id: string, + projectId: string, + patch: { name?: string; enabled?: boolean; fieldMapping?: ReceiverFieldMapping | null } + ): Promise { + const values: Record = {}; + if (patch.name !== undefined) values.name = patch.name; + if (patch.enabled !== undefined) values.enabled = patch.enabled; + if (patch.fieldMapping !== undefined) values.field_mapping = patch.fieldMapping; + if (Object.keys(values).length === 0) return false; + const result = await db + .updateTable('receivers') + .set(values) + .where('id', '=', id) + .where('project_id', '=', projectId) + .executeTakeFirst(); + return Number(result.numUpdatedRows || 0) > 0; + } + + async deleteReceiver(id: string, projectId: string): Promise { + const result = await db + .deleteFrom('receivers') + .where('id', '=', id) + .where('project_id', '=', projectId) + .executeTakeFirst(); + return Number(result.numDeletedRows || 0) > 0; + } + + /** Count receivers across all projects of an organization (capability limit input). */ + async countReceiversForOrg(organizationId: string): Promise { + const row = await db + .selectFrom('receivers') + .innerJoin('projects', 'projects.id', 'receivers.project_id') + .select((eb) => eb.fn.countAll().as('count')) + .where('projects.organization_id', '=', organizationId) + .executeTakeFirst(); + return Number(row?.count ?? 0); + } + + /** Load a receiver for the public ingest endpoint (org derived via projects join). */ + async getReceiverForIngest(receiverId: string): Promise { + const row = await db + .selectFrom('receivers') + .innerJoin('projects', 'projects.id', 'receivers.project_id') + .select([ + 'receivers.id', + 'receivers.project_id', + 'receivers.name', + 'receivers.adapter_type', + 'receivers.field_mapping', + 'receivers.enabled', + 'receivers.token_hash', + 'projects.organization_id', + ]) + .where('receivers.id', '=', receiverId) + .where('projects.deleted_at', 'is', null) + .executeTakeFirst(); + if (!row) return null; + return { + id: row.id, + projectId: row.project_id, + organizationId: row.organization_id, + name: row.name, + adapterType: row.adapter_type as ReceiverAdapterType, + fieldMapping: (row.field_mapping as ReceiverFieldMapping | null) ?? null, + enabled: row.enabled, + tokenHash: row.token_hash, + }; + } + + async recordEvent(receiverId: string, rawPayload: Record): Promise { + const result = await db + .insertInto('receiver_events') + .values({ receiver_id: receiverId, status: 'pending', raw_payload: rawPayload }) + .returning(['id']) + .executeTakeFirstOrThrow(); + return result.id; + } + + async getEventWithReceiver(eventId: string): Promise { + const row = await db + .selectFrom('receiver_events') + .innerJoin('receivers', 'receivers.id', 'receiver_events.receiver_id') + .innerJoin('projects', 'projects.id', 'receivers.project_id') + .select([ + 'receiver_events.id as event_id', + 'receiver_events.status', + 'receiver_events.raw_payload', + 'receivers.id', + 'receivers.project_id', + 'receivers.name', + 'receivers.adapter_type', + 'receivers.field_mapping', + 'receivers.enabled', + 'receivers.token_hash', + 'projects.organization_id', + ]) + .where('receiver_events.id', '=', eventId) + .where('projects.deleted_at', 'is', null) + .executeTakeFirst(); + if (!row) return null; + return { + eventId: row.event_id, + status: row.status as ReceiverEventStatus, + rawPayload: row.raw_payload as Record, + id: row.id, + projectId: row.project_id, + organizationId: row.organization_id, + name: row.name, + adapterType: row.adapter_type as ReceiverAdapterType, + fieldMapping: (row.field_mapping as ReceiverFieldMapping | null) ?? null, + enabled: row.enabled, + tokenHash: row.token_hash, + }; + } + + async completeEvent( + eventId: string, + outcome: { + status: Exclude; + normalized?: LogInput[] | null; + error?: string | null; + } + ): Promise { + await db + .updateTable('receiver_events') + .set({ + status: outcome.status, + // JSON.stringify because the pg driver would serialize a JS array root + // as a Postgres array literal, which is invalid for JSONB. + normalized: outcome.normalized != null ? JSON.stringify(outcome.normalized) : null, + error: outcome.error ?? null, + }) + .where('id', '=', eventId) + .execute(); + } + + async listEvents(receiverId: string, limit = 50): Promise { + const rows = await db + .selectFrom('receiver_events') + .select(['id', 'receiver_id', 'status', 'raw_payload', 'normalized', 'error', 'received_at']) + .where('receiver_id', '=', receiverId) + .orderBy('received_at', 'desc') + .limit(limit) + .execute(); + return rows.map((r) => ({ + id: r.id, + receiverId: r.receiver_id, + status: r.status as ReceiverEventStatus, + rawPayload: r.raw_payload as Record, + normalized: r.normalized ?? null, + error: r.error, + receivedAt: new Date(r.received_at), + })); + } + + async touchLastReceived(receiverId: string): Promise { + await db + .updateTable('receivers') + .set({ last_received_at: new Date() }) + .where('id', '=', receiverId) + .execute(); + } + + /** Keep only the newest `keep` events for a receiver. */ + async pruneEvents(receiverId: string, keep = ReceiversService.MAX_EVENTS_PER_RECEIVER): Promise { + await db + .deleteFrom('receiver_events') + .where('receiver_id', '=', receiverId) + .where('id', 'not in', (eb) => + eb + .selectFrom('receiver_events') + .select('id') + .where('receiver_id', '=', receiverId) + .orderBy('received_at', 'desc') + .limit(keep) + ) + .execute(); + } +} + +export const receiversService = new ReceiversService(); diff --git a/packages/backend/src/tests/modules/receivers/service.test.ts b/packages/backend/src/tests/modules/receivers/service.test.ts new file mode 100644 index 00000000..f0060659 --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/service.test.ts @@ -0,0 +1,120 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { db } from '../../../database/index.js'; +import { receiversService } from '../../../modules/receivers/service.js'; +import { createTestContext } from '../../helpers/factories.js'; + +describe('ReceiversService', () => { + let projectId: string; + let orgId: string; + + beforeEach(async () => { + await db.deleteFrom('receiver_events').execute(); + await db.deleteFrom('receivers').execute(); + await db.deleteFrom('api_keys').execute(); + await db.deleteFrom('organization_members').execute(); + await db.deleteFrom('projects').execute(); + await db.deleteFrom('organizations').execute(); + await db.deleteFrom('users').execute(); + const ctx = await createTestContext(); + projectId = ctx.project.id; + orgId = ctx.organization.id; + }); + + it('creates a receiver and returns the plaintext token once', async () => { + const { id, token } = await receiversService.createReceiver({ + projectId, + name: 'ci events', + adapterType: 'github', + }); + expect(token).toMatch(/^lr_[0-9a-f]{64}$/); + + const list = await receiversService.listReceivers(projectId); + expect(list).toHaveLength(1); + expect(list[0].id).toBe(id); + expect(list[0].adapterType).toBe('github'); + expect(list[0].enabled).toBe(true); + // plaintext token never stored + expect(JSON.stringify(list[0])).not.toContain(token); + }); + + it('verifies tokens with tokenMatches and rejects wrong tokens', async () => { + const { id, token } = await receiversService.createReceiver({ + projectId, + name: 'x', + adapterType: 'generic', + }); + const row = await receiversService.getReceiverForIngest(id); + expect(row).not.toBeNull(); + expect(row!.organizationId).toBe(orgId); + expect(receiversService.tokenMatches(token, row!.tokenHash)).toBe(true); + expect(receiversService.tokenMatches('lr_wrong', row!.tokenHash)).toBe(false); + }); + + it('updates and deletes only within the owning project', async () => { + const { id } = await receiversService.createReceiver({ + projectId, + name: 'x', + adapterType: 'generic', + }); + const other = await createTestContext(); + + expect(await receiversService.updateReceiver(id, other.project.id, { enabled: false })).toBe(false); + expect(await receiversService.updateReceiver(id, projectId, { enabled: false, name: 'y' })).toBe(true); + const list = await receiversService.listReceivers(projectId); + expect(list[0].enabled).toBe(false); + expect(list[0].name).toBe('y'); + + expect(await receiversService.deleteReceiver(id, other.project.id)).toBe(false); + expect(await receiversService.deleteReceiver(id, projectId)).toBe(true); + expect(await receiversService.listReceivers(projectId)).toHaveLength(0); + }); + + it('counts receivers across the org', async () => { + await receiversService.createReceiver({ projectId, name: 'a', adapterType: 'generic' }); + await receiversService.createReceiver({ projectId, name: 'b', adapterType: 'github' }); + expect(await receiversService.countReceiversForOrg(orgId)).toBe(2); + }); + + it('records, completes, lists and prunes events', async () => { + const { id } = await receiversService.createReceiver({ + projectId, + name: 'x', + adapterType: 'generic', + }); + + const eventId = await receiversService.recordEvent(id, { hello: 'world' }); + let joined = await receiversService.getEventWithReceiver(eventId); + expect(joined).not.toBeNull(); + expect(joined!.status).toBe('pending'); + expect(joined!.projectId).toBe(projectId); + + await receiversService.completeEvent(eventId, { + status: 'processed', + normalized: [{ time: new Date().toISOString(), service: 's', level: 'info', message: 'm' }], + }); + joined = await receiversService.getEventWithReceiver(eventId); + expect(joined!.status).toBe('processed'); + + const events = await receiversService.listEvents(id); + expect(events).toHaveLength(1); + expect((events[0].rawPayload as any).hello).toBe('world'); + + // prune: insert 5 more, keep 3 + for (let i = 0; i < 5; i++) { + await receiversService.recordEvent(id, { n: i }); + } + await receiversService.pruneEvents(id, 3); + expect(await receiversService.listEvents(id)).toHaveLength(3); + }); + + it('touches last_received_at', async () => { + const { id } = await receiversService.createReceiver({ + projectId, + name: 'x', + adapterType: 'generic', + }); + await receiversService.touchLastReceived(id); + const list = await receiversService.listReceivers(projectId); + expect(list[0].lastReceivedAt).not.toBeNull(); + }); +}); From 79e77aad47fdd8ef6e75583b973b6f54536e37dc Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:15:43 +0200 Subject: [PATCH 10/23] add receiver management routes with capability limit --- packages/backend/src/capabilities/registry.ts | 5 + .../backend/src/modules/audit-log/actions.ts | 3 + .../backend/src/modules/receivers/routes.ts | 202 ++++++++++++++++++ .../capabilities/receivers-limit.test.ts | 193 +++++++++++++++++ .../tests/modules/receivers/routes.test.ts | 135 ++++++++++++ 5 files changed, 538 insertions(+) create mode 100644 packages/backend/src/modules/receivers/routes.ts create mode 100644 packages/backend/src/tests/modules/capabilities/receivers-limit.test.ts create mode 100644 packages/backend/src/tests/modules/receivers/routes.test.ts diff --git a/packages/backend/src/capabilities/registry.ts b/packages/backend/src/capabilities/registry.ts index f2445b65..1d5ca1ca 100644 --- a/packages/backend/src/capabilities/registry.ts +++ b/packages/backend/src/capabilities/registry.ts @@ -94,6 +94,11 @@ export const CAPABILITIES = { defaultLimit: null, description: 'Maximum custom dashboards per organization', }, + 'receivers.max': { + kind: 'limit', + defaultLimit: null, + description: 'Maximum inbound webhook receivers per organization', + }, // Consumption quotas (OSS-permissive: null = unlimited). signal maps to #212 metering types. 'ingestion.max_bytes_monthly': { diff --git a/packages/backend/src/modules/audit-log/actions.ts b/packages/backend/src/modules/audit-log/actions.ts index 135684cb..c6dca7fc 100644 --- a/packages/backend/src/modules/audit-log/actions.ts +++ b/packages/backend/src/modules/audit-log/actions.ts @@ -23,6 +23,9 @@ export const AUDIT_ACTIONS = { // api keys 'apikey.created': 'config_change', 'apikey.revoked': 'config_change', + // inbound webhook receivers (#155) + 'receiver.created': 'config_change', + 'receiver.deleted': 'config_change', // users and membership 'user.registered': 'user_management', 'user.created': 'user_management', diff --git a/packages/backend/src/modules/receivers/routes.ts b/packages/backend/src/modules/receivers/routes.ts new file mode 100644 index 00000000..a23c7bf7 --- /dev/null +++ b/packages/backend/src/modules/receivers/routes.ts @@ -0,0 +1,202 @@ +import type { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { receiverAdapterTypeSchema, receiverFieldMappingSchema } from '@logtide/shared'; +import { context } from '@logtide/shared/context'; +import { receiversService } from './service.js'; +import { authenticate } from '../auth/middleware.js'; +import { projectsService } from '../projects/service.js'; +import { auditLogService } from '../audit-log/index.js'; +import { assertWithinLimit, withLimitLock } from '../../capabilities/index.js'; +import { CapabilityError } from '../../capabilities/errors.js'; + +const createReceiverSchema = z + .object({ + name: z.string().min(1, 'Name is required').max(100, 'Name too long'), + adapterType: receiverAdapterTypeSchema, + fieldMapping: receiverFieldMappingSchema.optional().nullable(), + }) + .refine((v) => v.adapterType === 'generic' || v.fieldMapping == null, { + message: 'fieldMapping is only supported by the generic adapter', + path: ['fieldMapping'], + }); + +const updateReceiverSchema = z + .object({ + name: z.string().min(1).max(100).optional(), + enabled: z.boolean().optional(), + fieldMapping: receiverFieldMappingSchema.optional().nullable(), + }) + .refine((v) => v.name !== undefined || v.enabled !== undefined || v.fieldMapping !== undefined, { + message: 'At least one field must be provided', + }); + +const projectIdSchema = z.object({ + projectId: z.string().uuid('Invalid project ID format'), +}); + +const receiverIdSchema = z.object({ + projectId: z.string().uuid('Invalid project ID format'), + id: z.string().uuid('Invalid receiver ID format'), +}); + +const eventsQuerySchema = z.object({ + limit: z.coerce.number().int().min(1).max(100).default(50), +}); + +export async function receiversRoutes(fastify: FastifyInstance) { + // All routes require authentication + fastify.addHook('onRequest', authenticate); + + // List receivers for a project + fastify.get('/:projectId/receivers', async (request: any, reply) => { + try { + const { projectId } = projectIdSchema.parse(request.params); + + const project = await projectsService.getProjectById(projectId, request.user.id); + if (!project || project.deletedAt) { + return reply.status(404).send({ error: 'Project not found or access denied' }); + } + + const receivers = await receiversService.listReceivers(projectId); + return reply.send({ receivers }); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Invalid project ID format' }); + } + throw error; + } + }); + + // Create a receiver + fastify.post('/:projectId/receivers', async (request: any, reply) => { + try { + const { projectId } = projectIdSchema.parse(request.params); + const body = createReceiverSchema.parse(request.body); + + const project = await projectsService.getProjectById(projectId, request.user.id); + if (!project || project.deletedAt) { + return reply.status(404).send({ error: 'Project not found or access denied' }); + } + + const result = await withLimitLock(project.organizationId, 'receivers.max', async () => { + await context.runAsSystem('receivers:create-limit-check', async () => { + await context.with({ organizationId: project.organizationId }, async () => { + const count = await receiversService.countReceiversForOrg(project.organizationId); + await assertWithinLimit('receivers.max', count); + }); + }); + + return receiversService.createReceiver({ + projectId, + name: body.name, + adapterType: body.adapterType, + fieldMapping: body.fieldMapping ?? null, + }); + }); + + await auditLogService.record({ + action: 'receiver.created', + target: { type: 'receiver', id: result.id }, + organizationId: project.organizationId, + metadata: { name: body.name, adapterType: body.adapterType, projectId }, + }); + + return reply.status(201).send({ + id: result.id, + token: result.token, + ingestPath: `/api/v1/receivers/${result.id}/${result.token}`, + message: 'Receiver created. Save the URL securely - the token will not be shown again.', + }); + } catch (error) { + if (error instanceof CapabilityError) { + throw error; + } + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Validation error', details: error.errors }); + } + throw error; + } + }); + + // Update a receiver (rename, enable/disable, edit field mapping) + fastify.patch('/:projectId/receivers/:id', async (request: any, reply) => { + try { + const { projectId, id } = receiverIdSchema.parse(request.params); + const body = updateReceiverSchema.parse(request.body); + + const project = await projectsService.getProjectById(projectId, request.user.id); + if (!project || project.deletedAt) { + return reply.status(404).send({ error: 'Project not found or access denied' }); + } + + const updated = await receiversService.updateReceiver(id, projectId, body); + if (!updated) { + return reply.status(404).send({ error: 'Receiver not found' }); + } + return reply.send({ success: true }); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Validation error', details: error.errors }); + } + throw error; + } + }); + + // Delete a receiver + fastify.delete('/:projectId/receivers/:id', async (request: any, reply) => { + try { + const { projectId, id } = receiverIdSchema.parse(request.params); + + const project = await projectsService.getProjectById(projectId, request.user.id); + if (!project || project.deletedAt) { + return reply.status(404).send({ error: 'Project not found or access denied' }); + } + + const deleted = await receiversService.deleteReceiver(id, projectId); + if (!deleted) { + return reply.status(404).send({ error: 'Receiver not found' }); + } + + await auditLogService.record({ + action: 'receiver.deleted', + target: { type: 'receiver', id }, + organizationId: project.organizationId, + metadata: { projectId }, + }); + + return reply.status(204).send(); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Invalid ID format' }); + } + throw error; + } + }); + + // Recent events for a receiver (debug view) + fastify.get('/:projectId/receivers/:id/events', async (request: any, reply) => { + try { + const { projectId, id } = receiverIdSchema.parse(request.params); + const { limit } = eventsQuerySchema.parse(request.query ?? {}); + + const project = await projectsService.getProjectById(projectId, request.user.id); + if (!project || project.deletedAt) { + return reply.status(404).send({ error: 'Project not found or access denied' }); + } + + // Confirm the receiver belongs to this project before listing its events. + const receivers = await receiversService.listReceivers(projectId); + if (!receivers.some((r) => r.id === id)) { + return reply.status(404).send({ error: 'Receiver not found' }); + } + + const events = await receiversService.listEvents(id, limit); + return reply.send({ events }); + } catch (error) { + if (error instanceof z.ZodError) { + return reply.status(400).send({ error: 'Invalid ID format' }); + } + throw error; + } + }); +} diff --git a/packages/backend/src/tests/modules/capabilities/receivers-limit.test.ts b/packages/backend/src/tests/modules/capabilities/receivers-limit.test.ts new file mode 100644 index 00000000..1b1ba306 --- /dev/null +++ b/packages/backend/src/tests/modules/capabilities/receivers-limit.test.ts @@ -0,0 +1,193 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import Fastify, { FastifyInstance } from 'fastify'; +import crypto from 'crypto'; +import { db } from '../../../database/index.js'; +import { receiversRoutes } from '../../../modules/receivers/routes.js'; +import { contextPlugin } from '../../../context/index.js'; +import { capabilities } from '../../../capabilities/index.js'; +import { createTestContext } from '../../helpers/factories.js'; + +async function createTestSession(userId: string) { + const token = crypto.randomBytes(32).toString('hex'); + await db + .insertInto('sessions') + .values({ user_id: userId, token, expires_at: new Date(Date.now() + 24 * 60 * 60 * 1000) }) + .execute(); + return token; +} + +async function insertReceiver(projectId: string, n = 1) { + const tokenHash = crypto + .createHash('sha256') + .update(`lr_test_${crypto.randomBytes(16).toString('hex')}`) + .digest('hex'); + const [receiver] = await db + .insertInto('receivers') + .values({ project_id: projectId, name: `Receiver ${n}`, adapter_type: 'generic', token_hash: tokenHash }) + .returningAll() + .execute(); + return receiver; +} + +describe('receivers.max enforcement', () => { + let app: FastifyInstance; + let orgId: string; + let userId: string; + let projectId: string; + let token: string; + + beforeAll(async () => { + app = Fastify(); + await app.register(contextPlugin); + await app.register(receiversRoutes, { prefix: '/api/v1/projects' }); + await app.ready(); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + await db.deleteFrom('organization_entitlements').execute(); + await db.deleteFrom('receiver_events').execute(); + await db.deleteFrom('receivers').execute(); + await db.deleteFrom('api_keys').execute(); + await db.deleteFrom('sessions').execute(); + await db.deleteFrom('organization_members').execute(); + await db.deleteFrom('projects').execute(); + await db.deleteFrom('organizations').execute(); + await db.deleteFrom('users').execute(); + + const ctx = await createTestContext(); + orgId = ctx.organization.id; + userId = ctx.user.id; + projectId = ctx.project.id; + token = await createTestSession(userId); + capabilities.invalidate(orgId); + }); + + // Case 1: org-wide blocking across projects + it('blocks receiver creation when the org-wide limit is reached (receiver on a different project)', async () => { + await db + .insertInto('organization_entitlements') + .values({ organization_id: orgId, capability: 'receivers.max', enabled: null, limit_value: 1 }) + .execute(); + capabilities.invalidate(orgId); + + const project2 = await db + .insertInto('projects') + .values({ + name: 'Project Two', + slug: `proj2-${Date.now()}`, + organization_id: orgId, + user_id: userId, + }) + .returningAll() + .executeTakeFirstOrThrow(); + + await insertReceiver(projectId, 1); + + const res = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${project2.id}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'New Receiver', adapterType: 'generic' }, + }); + + expect(res.statusCode).toBe(403); + const body = JSON.parse(res.body); + expect(body.statusCode).toBe(403); + expect(body.code).toBe('capability.receivers.max.limit_reached'); + }); + + // Case 2: under limit => passes + it('allows receiver creation when under the limit', async () => { + await db + .insertInto('organization_entitlements') + .values({ organization_id: orgId, capability: 'receivers.max', enabled: null, limit_value: 2 }) + .execute(); + capabilities.invalidate(orgId); + + await insertReceiver(projectId, 1); + + const res = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'New Receiver', adapterType: 'generic' }, + }); + + expect(res.statusCode).toBe(201); + }); + + // Case 3: unlimited default (no entitlement row) => passes + it('allows receiver creation when no limit is configured (unlimited default)', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'New Receiver', adapterType: 'generic' }, + }); + + expect(res.statusCode).toBe(201); + }); + + // Case 5: concurrent creates must not race past a finite limit + it('serializes concurrent creates so the limit is never exceeded (race)', async () => { + await db + .insertInto('organization_entitlements') + .values({ organization_id: orgId, capability: 'receivers.max', enabled: null, limit_value: 2 }) + .execute(); + capabilities.invalidate(orgId); + + const attempts = 6; + const results = await Promise.all( + Array.from({ length: attempts }, (_, i) => + app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: `Race Receiver ${i}`, adapterType: 'generic' }, + }) + ) + ); + + const created = results.filter((r) => r.statusCode === 201).length; + const blocked = results.filter((r) => r.statusCode === 403).length; + + expect(created).toBe(2); + expect(blocked).toBe(attempts - 2); + + // Hard invariant: the org never ends up over its configured limit. + const total = await db + .selectFrom('receivers') + .innerJoin('projects', 'projects.id', 'receivers.project_id') + .select((eb) => eb.fn.countAll().as('c')) + .where('projects.organization_id', '=', orgId) + .executeTakeFirstOrThrow(); + expect(Number(total.c)).toBe(2); + }); + + // Case 4: org isolation => org A at limit does not block org B + it('does not block org B when org A is at the limit', async () => { + await db + .insertInto('organization_entitlements') + .values({ organization_id: orgId, capability: 'receivers.max', enabled: null, limit_value: 1 }) + .execute(); + capabilities.invalidate(orgId); + await insertReceiver(projectId, 1); + + const ctxB = await createTestContext(); + const tokenB = await createTestSession(ctxB.user.id); + capabilities.invalidate(ctxB.organization.id); + + const res = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${ctxB.project.id}/receivers`, + headers: { Authorization: `Bearer ${tokenB}` }, + payload: { name: 'New Receiver', adapterType: 'generic' }, + }); + + expect(res.statusCode).toBe(201); + }); +}); diff --git a/packages/backend/src/tests/modules/receivers/routes.test.ts b/packages/backend/src/tests/modules/receivers/routes.test.ts new file mode 100644 index 00000000..bd98ee4a --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/routes.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import Fastify, { FastifyInstance } from 'fastify'; +import crypto from 'crypto'; +import { db } from '../../../database/index.js'; +import { receiversRoutes } from '../../../modules/receivers/routes.js'; +import { contextPlugin } from '../../../context/index.js'; +import { createTestContext } from '../../helpers/factories.js'; +import { receiversService } from '../../../modules/receivers/service.js'; + +async function createTestSession(userId: string) { + const token = crypto.randomBytes(32).toString('hex'); + await db + .insertInto('sessions') + .values({ user_id: userId, token, expires_at: new Date(Date.now() + 24 * 60 * 60 * 1000) }) + .execute(); + return token; +} + +describe('receivers management routes', () => { + let app: FastifyInstance; + let projectId: string; + let userId: string; + let token: string; + + beforeAll(async () => { + app = Fastify(); + await app.register(contextPlugin); + await app.register(receiversRoutes, { prefix: '/api/v1/projects' }); + await app.ready(); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + await db.deleteFrom('receiver_events').execute(); + await db.deleteFrom('receivers').execute(); + await db.deleteFrom('sessions').execute(); + await db.deleteFrom('api_keys').execute(); + await db.deleteFrom('organization_members').execute(); + await db.deleteFrom('projects').execute(); + await db.deleteFrom('organizations').execute(); + await db.deleteFrom('users').execute(); + const ctx = await createTestContext(); + projectId = ctx.project.id; + userId = ctx.user.id; + token = await createTestSession(userId); + }); + + it('creates a receiver and returns token and ingest path once', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'ci', adapterType: 'github' }, + }); + expect(res.statusCode).toBe(201); + const body = JSON.parse(res.body); + expect(body.token).toMatch(/^lr_[0-9a-f]{64}$/); + expect(body.ingestPath).toBe(`/api/v1/receivers/${body.id}/${body.token}`); + }); + + it('rejects fieldMapping on non-generic adapters and validates mapping shape', async () => { + const bad = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'x', adapterType: 'github', fieldMapping: { message: 'a' } }, + }); + expect(bad.statusCode).toBe(400); + + const badMapping = await app.inject({ + method: 'POST', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + payload: { name: 'x', adapterType: 'generic', fieldMapping: { nope: 'a' } }, + }); + expect(badMapping.statusCode).toBe(400); + }); + + it('lists receivers without exposing token hashes', async () => { + await receiversService.createReceiver({ projectId, name: 'a', adapterType: 'generic' }); + const res = await app.inject({ + method: 'GET', + url: `/api/v1/projects/${projectId}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + }); + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body); + expect(body.receivers).toHaveLength(1); + expect(res.body).not.toContain('token'); + }); + + it('updates and deletes receivers', async () => { + const { id } = await receiversService.createReceiver({ projectId, name: 'a', adapterType: 'generic' }); + + const patch = await app.inject({ + method: 'PATCH', + url: `/api/v1/projects/${projectId}/receivers/${id}`, + headers: { Authorization: `Bearer ${token}` }, + payload: { enabled: false }, + }); + expect(patch.statusCode).toBe(200); + + const del = await app.inject({ + method: 'DELETE', + url: `/api/v1/projects/${projectId}/receivers/${id}`, + headers: { Authorization: `Bearer ${token}` }, + }); + expect(del.statusCode).toBe(204); + }); + + it('lists recent events', async () => { + const { id } = await receiversService.createReceiver({ projectId, name: 'a', adapterType: 'generic' }); + await receiversService.recordEvent(id, { hello: 1 }); + const res = await app.inject({ + method: 'GET', + url: `/api/v1/projects/${projectId}/receivers/${id}/events`, + headers: { Authorization: `Bearer ${token}` }, + }); + expect(res.statusCode).toBe(200); + expect(JSON.parse(res.body).events).toHaveLength(1); + }); + + it('denies access to projects of other users (404)', async () => { + const other = await createTestContext(); + const res = await app.inject({ + method: 'GET', + url: `/api/v1/projects/${other.project.id}/receivers`, + headers: { Authorization: `Bearer ${token}` }, + }); + expect(res.statusCode).toBe(404); + }); +}); From 67271bc98f79fe1eb099fac33c50d74cca7e3a54 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:19:04 +0200 Subject: [PATCH 11/23] add public receiver endpoint and event processing job --- packages/backend/src/modules/auth/plugin.ts | 5 +- .../src/modules/receivers/ingest-routes.ts | 57 +++++++ .../backend/src/queue/jobs/receiver-event.ts | 116 ++++++++++++++ packages/backend/src/server.ts | 4 + .../tests/modules/receivers/ingest.test.ts | 150 ++++++++++++++++++ packages/backend/src/worker.ts | 18 +++ 6 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/modules/receivers/ingest-routes.ts create mode 100644 packages/backend/src/queue/jobs/receiver-event.ts create mode 100644 packages/backend/src/tests/modules/receivers/ingest.test.ts diff --git a/packages/backend/src/modules/auth/plugin.ts b/packages/backend/src/modules/auth/plugin.ts index f4241af7..dc056447 100644 --- a/packages/backend/src/modules/auth/plugin.ts +++ b/packages/backend/src/modules/auth/plugin.ts @@ -87,7 +87,10 @@ const authPlugin: FastifyPluginAsync = async (fastify) => { request.url.startsWith('/api/v1/invitations') || request.url.startsWith('/api/v1/status') || request.url.startsWith('/api/v1/status-incidents') || - request.url.startsWith('/api/v1/maintenances') + request.url.startsWith('/api/v1/maintenances') || + // Inbound webhook receivers (#155): the URL token is the credential, + // verified by the route itself. + request.url.startsWith('/api/v1/receivers') ) { return; } diff --git a/packages/backend/src/modules/receivers/ingest-routes.ts b/packages/backend/src/modules/receivers/ingest-routes.ts new file mode 100644 index 00000000..550e09f4 --- /dev/null +++ b/packages/backend/src/modules/receivers/ingest-routes.ts @@ -0,0 +1,57 @@ +import type { FastifyInstance } from 'fastify'; +import { z } from 'zod'; +import { receiversService } from './service.js'; +import { createQueue } from '../../queue/connection.js'; +import { RECEIVER_EVENTS_QUEUE, type ReceiverEventJobData } from '../../queue/jobs/receiver-event.js'; + +const paramsSchema = z.object({ + receiverId: z.string().uuid(), + token: z.string().min(1).max(200), +}); + +/** Serialized raw payloads above this size are rejected (protects receiver_events). */ +const MAX_PAYLOAD_BYTES = 256 * 1024; + +const receiverEventsQueue = createQueue(RECEIVER_EVENTS_QUEUE); + +/** + * Public inbound webhook endpoint (#155): POST /api/v1/receivers/:receiverId/:token + * Token authentication happens here (the URL is the credential; GitHub and + * Uptime Robot cannot send custom headers). The auth plugin skips this prefix. + */ +export async function receiversIngestRoutes(fastify: FastifyInstance) { + fastify.post('/:receiverId/:token', async (request, reply) => { + const params = paramsSchema.safeParse(request.params); + if (!params.success) { + return reply.status(404).send({ error: 'Receiver not found' }); + } + const { receiverId, token } = params.data; + + const receiver = await receiversService.getReceiverForIngest(receiverId); + if (!receiver) { + return reply.status(404).send({ error: 'Receiver not found' }); + } + if (!receiversService.tokenMatches(token, receiver.tokenHash)) { + return reply.status(401).send({ error: 'Invalid receiver token' }); + } + if (!receiver.enabled) { + return reply.status(403).send({ error: 'Receiver is disabled' }); + } + + const payload = request.body; + if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) { + return reply.status(400).send({ error: 'Payload must be a JSON object' }); + } + if (Buffer.byteLength(JSON.stringify(payload), 'utf8') > MAX_PAYLOAD_BYTES) { + return reply.status(413).send({ error: 'Payload too large (max 256 KB)' }); + } + + const eventId = await receiversService.recordEvent( + receiverId, + payload as Record + ); + await receiverEventsQueue.add('process', { eventId }, { maxAttempts: 1 }); + + return reply.status(202).send({ eventId }); + }); +} diff --git a/packages/backend/src/queue/jobs/receiver-event.ts b/packages/backend/src/queue/jobs/receiver-event.ts new file mode 100644 index 00000000..b3cdf720 --- /dev/null +++ b/packages/backend/src/queue/jobs/receiver-event.ts @@ -0,0 +1,116 @@ +/** + * Inbound receiver event processing (#155). The public endpoint stores the raw + * payload and enqueues this job; here the adapter normalizes it and the result + * goes through the normal ingestion pipeline (PII masking, quotas, Sigma). + * Failures are recorded on the event row for the UI; no automatic retry. + */ +import { logSchema, type LogInput } from '@logtide/shared'; +import { context } from '@logtide/shared/context'; +import type { IJob } from '../abstractions/types.js'; +import { receiversService, ReceiversService } from '../../modules/receivers/service.js'; +import { getAdapter, type AdapterResult } from '../../modules/receivers/adapters/index.js'; +import { ingestionService } from '../../modules/ingestion/service.js'; + +export const RECEIVER_EVENTS_QUEUE = 'receiver-events'; + +export interface ReceiverEventJobData { + eventId: string; +} + +export async function processReceiverEvent(job: IJob): Promise { + const event = await receiversService.getEventWithReceiver(job.data.eventId); + if (!event) { + console.warn(`[ReceiverEvent] event ${job.data.eventId} not found, skipping`); + return; + } + if (event.status !== 'pending') { + return; // idempotent: already terminal + } + + let result: AdapterResult; + try { + const adapter = getAdapter(event.adapterType); + result = adapter(event.rawPayload, { + id: event.id, + name: event.name, + adapterType: event.adapterType, + fieldMapping: event.fieldMapping, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await receiversService.completeEvent(event.eventId, { + status: 'failed', + error: `adapter error: ${message}`, + }); + return; + } + + if (result.kind === 'skipped') { + await receiversService.completeEvent(event.eventId, { + status: 'skipped', + error: result.reason, + }); + await finalize(event.id); + return; + } + + const valid: LogInput[] = []; + const invalid: string[] = []; + for (const log of result.logs) { + const parsed = logSchema.safeParse(log); + if (parsed.success) valid.push(parsed.data); + else invalid.push(parsed.error.errors.map((e) => e.message).join('; ')); + } + + if (valid.length === 0) { + await receiversService.completeEvent(event.eventId, { + status: 'failed', + normalized: result.logs, + error: `adapter produced no valid logs: ${invalid.join(' | ')}`, + }); + await finalize(event.id); + return; + } + + try { + const ingestResult = await context.runAsSystem('receiver-event', () => + context.with( + { organizationId: event.organizationId, projectId: event.projectId }, + () => ingestionService.ingestLogs(valid, event.projectId) + ) + ); + + if (ingestResult.received === 0) { + const reasons = ingestResult.rejected.map((r) => r.reason).join('; '); + await receiversService.completeEvent(event.eventId, { + status: 'failed', + normalized: valid, + error: `all logs rejected by ingestion: ${reasons}`, + }); + } else { + const partial = + ingestResult.rejected.length > 0 + ? `${ingestResult.rejected.length} of ${result.logs.length} logs rejected` + : null; + await receiversService.completeEvent(event.eventId, { + status: 'processed', + normalized: valid, + error: partial, + }); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await receiversService.completeEvent(event.eventId, { + status: 'failed', + normalized: valid, + error: `ingestion failed: ${message}`, + }); + } + + await finalize(event.id); +} + +async function finalize(receiverId: string): Promise { + await receiversService.touchLastReceived(receiverId); + await receiversService.pruneEvents(receiverId, ReceiversService.MAX_EVENTS_PER_RECEIVER); +} diff --git a/packages/backend/src/server.ts b/packages/backend/src/server.ts index a20173fb..1733622a 100644 --- a/packages/backend/src/server.ts +++ b/packages/backend/src/server.ts @@ -17,6 +17,8 @@ import { organizationsRoutes } from './modules/organizations/routes.js'; import { invitationsRoutes } from './modules/invitations/routes.js'; import { notificationsRoutes } from './modules/notifications/routes.js'; import { apiKeysRoutes } from './modules/api-keys/routes.js'; +import { receiversRoutes } from './modules/receivers/routes.js'; +import { receiversIngestRoutes } from './modules/receivers/ingest-routes.js'; import dashboardRoutes from './modules/dashboard/routes.js'; import { sigmaRoutes } from './modules/sigma/routes.js'; import { siemRoutes } from './modules/siem/routes.js'; @@ -189,6 +191,8 @@ export async function build(opts = {}) { await fastify.register(registerSiemSseRoutes); await fastify.register(exceptionsRoutes); await fastify.register(apiKeysRoutes, { prefix: '/api/v1/projects' }); + await fastify.register(receiversRoutes, { prefix: '/api/v1/projects' }); + await fastify.register(receiversIngestRoutes, { prefix: '/api/v1/receivers' }); await fastify.register(dashboardRoutes); await fastify.register(adminRoutes, { prefix: '/api/v1/admin' }); await fastify.register(settingsRoutes, { prefix: '/api/v1/admin/settings' }); diff --git a/packages/backend/src/tests/modules/receivers/ingest.test.ts b/packages/backend/src/tests/modules/receivers/ingest.test.ts new file mode 100644 index 00000000..9af26870 --- /dev/null +++ b/packages/backend/src/tests/modules/receivers/ingest.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import Fastify, { FastifyInstance } from 'fastify'; +import { db } from '../../../database/index.js'; +import { contextPlugin } from '../../../context/index.js'; +import { receiversIngestRoutes } from '../../../modules/receivers/ingest-routes.js'; +import { receiversService } from '../../../modules/receivers/service.js'; +import { processReceiverEvent } from '../../../queue/jobs/receiver-event.js'; +import type { IJob } from '../../../queue/abstractions/types.js'; +import type { ReceiverEventJobData } from '../../../queue/jobs/receiver-event.js'; +import { createTestContext } from '../../helpers/factories.js'; + +function fakeJob(eventId: string): IJob { + return { id: 'test-job', name: 'process', data: { eventId } } as IJob; +} + +describe('receiver ingest endpoint', () => { + let app: FastifyInstance; + let projectId: string; + let receiverId: string; + let token: string; + + beforeAll(async () => { + app = Fastify(); + await app.register(contextPlugin); + await app.register(receiversIngestRoutes, { prefix: '/api/v1/receivers' }); + await app.ready(); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + await db.deleteFrom('receiver_events').execute(); + await db.deleteFrom('receivers').execute(); + await db.deleteFrom('api_keys').execute(); + await db.deleteFrom('organization_members').execute(); + await db.deleteFrom('projects').execute(); + await db.deleteFrom('organizations').execute(); + await db.deleteFrom('users').execute(); + const ctx = await createTestContext(); + projectId = ctx.project.id; + const created = await receiversService.createReceiver({ + projectId, + name: 'generic in', + adapterType: 'generic', + }); + receiverId = created.id; + token = created.token; + }); + + it('accepts a valid payload with 202 and records a pending event', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/${token}`, + payload: { msg: 'hello' }, + }); + expect(res.statusCode).toBe(202); + const { eventId } = JSON.parse(res.body); + const event = await receiversService.getEventWithReceiver(eventId); + expect(event).not.toBeNull(); + expect(event!.status).toBe('pending'); + }); + + it('rejects wrong tokens with 401 and unknown receivers with 404', async () => { + const bad = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/lr_${'0'.repeat(64)}`, + payload: { a: 1 }, + }); + expect(bad.statusCode).toBe(401); + + const missing = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/00000000-0000-4000-8000-000000000000/${token}`, + payload: { a: 1 }, + }); + expect(missing.statusCode).toBe(404); + }); + + it('rejects disabled receivers with 403', async () => { + await receiversService.updateReceiver(receiverId, projectId, { enabled: false }); + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/${token}`, + payload: { a: 1 }, + }); + expect(res.statusCode).toBe(403); + }); + + it('rejects non-object payloads with 400', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/${token}`, + payload: JSON.stringify([1, 2, 3]), + headers: { 'content-type': 'application/json' }, + }); + expect(res.statusCode).toBe(400); + }); + + it('processes an event end to end: logs are ingested, event marked processed', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/${token}`, + payload: { msg: 'from external' }, + }); + const { eventId } = JSON.parse(res.body); + + await processReceiverEvent(fakeJob(eventId)); + + const event = await receiversService.getEventWithReceiver(eventId); + expect(event!.status).toBe('processed'); + + const events = await receiversService.listEvents(receiverId, 10); + expect(events[0].normalized).not.toBeNull(); + + const receivers = await receiversService.listReceivers(projectId); + expect(receivers[0].lastReceivedAt).not.toBeNull(); + }); + + it('marks github ping events as skipped', async () => { + const gh = await receiversService.createReceiver({ + projectId, + name: 'gh', + adapterType: 'github', + }); + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${gh.id}/${gh.token}`, + payload: { zen: 'Design for failure.', hook_id: 1 }, + }); + const { eventId } = JSON.parse(res.body); + await processReceiverEvent(fakeJob(eventId)); + const event = await receiversService.getEventWithReceiver(eventId); + expect(event!.status).toBe('skipped'); + }); + + it('is idempotent: reprocessing a non-pending event is a no-op', async () => { + const res = await app.inject({ + method: 'POST', + url: `/api/v1/receivers/${receiverId}/${token}`, + payload: { msg: 'x' }, + }); + const { eventId } = JSON.parse(res.body); + await processReceiverEvent(fakeJob(eventId)); + await processReceiverEvent(fakeJob(eventId)); // must not throw or double-ingest + const event = await receiversService.getEventWithReceiver(eventId); + expect(event!.status).toBe('processed'); + }); +}); diff --git a/packages/backend/src/worker.ts b/packages/backend/src/worker.ts index 9daf3dee..e7efc061 100644 --- a/packages/backend/src/worker.ts +++ b/packages/backend/src/worker.ts @@ -17,6 +17,7 @@ import { processDigestGeneration } from './queue/jobs/digest-generation.js'; import type { DigestJobPayload } from './modules/digests/scheduler.js'; import { processWebhookDelivery } from './queue/jobs/webhook-delivery.js'; import type { WebhookDeliveryJobData } from './modules/webhooks/types.js'; +import { processReceiverEvent, type ReceiverEventJobData } from './queue/jobs/receiver-event.js'; import { alertsService } from './modules/alerts/index.js'; import { monitorService } from './modules/monitoring/index.js'; import { maintenanceService } from './modules/maintenances/service.js'; @@ -103,6 +104,11 @@ const webhookDeliveryWorker = createWorker('webhook-deli await processWebhookDelivery(job); }); +// Create worker for inbound receiver events (#155) +const receiverEventsWorker = createWorker('receiver-events', async (job) => { + await processReceiverEvent(job); +}); + // Start workers (required for graphile-worker backend, no-op for BullMQ) console.log(`[Worker] Using queue backend: ${getQueueBackend()}`); await startQueueWorkers(); @@ -340,6 +346,17 @@ webhookDeliveryWorker.on('failed', (job, err) => { } }); +receiverEventsWorker.on('failed', (job, err) => { + console.error(`Receiver event job ${job?.id} failed:`, err); + if (isInternalLoggingEnabled()) { + hub.captureLog('error', `Receiver event job failed: ${err.message}`, { + error: { name: err.name, message: err.message, stack: err.stack }, + jobId: job?.id, + eventId: job?.data?.eventId, + }); + } +}); + // Lock to prevent overlapping alert checks (race condition protection) let isCheckingAlerts = false; @@ -896,6 +913,7 @@ async function gracefulShutdown(signal: string) { await pipelineWorker.close(); await digestWorker.close(); await webhookDeliveryWorker.close(); + await receiverEventsWorker.close(); console.log('[Worker] Workers closed'); // Close queue system (Redis/PostgreSQL connections) From 98cbe93019a36e3d39e913b7e2108ba34bdfd2a5 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:24:02 +0200 Subject: [PATCH 12/23] add receiver management ui in project settings --- packages/frontend/src/lib/api/receivers.ts | 121 ++++++++ .../receivers/CreateReceiverDialog.svelte | 273 ++++++++++++++++++ .../receivers/ReceiverEventsDialog.svelte | 128 ++++++++ .../src/lib/components/receivers/index.ts | 2 + .../projects/[id]/settings/+page.svelte | 201 ++++++++++++- 5 files changed, 724 insertions(+), 1 deletion(-) create mode 100644 packages/frontend/src/lib/api/receivers.ts create mode 100644 packages/frontend/src/lib/components/receivers/CreateReceiverDialog.svelte create mode 100644 packages/frontend/src/lib/components/receivers/ReceiverEventsDialog.svelte create mode 100644 packages/frontend/src/lib/components/receivers/index.ts diff --git a/packages/frontend/src/lib/api/receivers.ts b/packages/frontend/src/lib/api/receivers.ts new file mode 100644 index 00000000..45a2cd51 --- /dev/null +++ b/packages/frontend/src/lib/api/receivers.ts @@ -0,0 +1,121 @@ +import { getApiBaseUrl } from '$lib/config'; +import { getAuthToken } from '$lib/utils/auth'; + +export type ReceiverAdapterType = 'github' | 'uptime' | 'generic'; +export type ReceiverEventStatus = 'pending' | 'processed' | 'skipped' | 'failed'; + +export interface ReceiverFieldMapping { + message?: string; + level?: string; + service?: string; + timestamp?: string; + levelMap?: Record; + defaults?: { level?: string; service?: string }; +} + +export interface Receiver { + id: string; + projectId: string; + name: string; + adapterType: ReceiverAdapterType; + fieldMapping: ReceiverFieldMapping | null; + enabled: boolean; + createdAt: string; + lastReceivedAt: string | null; +} + +export interface ReceiverEvent { + id: string; + receiverId: string; + status: ReceiverEventStatus; + rawPayload: Record; + normalized: unknown | null; + error: string | null; + receivedAt: string; +} + +export interface CreateReceiverInput { + name: string; + adapterType: ReceiverAdapterType; + fieldMapping?: ReceiverFieldMapping | null; +} + +export interface CreateReceiverResponse { + id: string; + token: string; + ingestPath: string; + message: string; +} + +export class ReceiversAPI { + constructor(private getToken: () => string | null) {} + + private async request(path: string, options: RequestInit = {}): Promise { + const token = this.getToken(); + if (!token) { + throw new Error('Not authenticated'); + } + + return fetch(`${getApiBaseUrl()}${path}`, { + ...options, + headers: { + ...(options.body ? { 'Content-Type': 'application/json' } : {}), + Authorization: `Bearer ${token}`, + ...options.headers, + }, + }); + } + + async list(projectId: string): Promise<{ receivers: Receiver[] }> { + const response = await this.request(`/projects/${projectId}/receivers`); + if (!response.ok) { + throw new Error('Failed to fetch receivers'); + } + return response.json(); + } + + async create(projectId: string, input: CreateReceiverInput): Promise { + const response = await this.request(`/projects/${projectId}/receivers`, { + method: 'POST', + body: JSON.stringify(input), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || 'Failed to create receiver'); + } + return response.json(); + } + + async update( + projectId: string, + id: string, + patch: { name?: string; enabled?: boolean; fieldMapping?: ReceiverFieldMapping | null } + ): Promise { + const response = await this.request(`/projects/${projectId}/receivers/${id}`, { + method: 'PATCH', + body: JSON.stringify(patch), + }); + if (!response.ok) { + throw new Error('Failed to update receiver'); + } + } + + async delete(projectId: string, id: string): Promise { + const response = await this.request(`/projects/${projectId}/receivers/${id}`, { + method: 'DELETE', + }); + if (!response.ok && response.status !== 204) { + throw new Error('Failed to delete receiver'); + } + } + + async listEvents(projectId: string, id: string, limit = 50): Promise<{ events: ReceiverEvent[] }> { + const response = await this.request(`/projects/${projectId}/receivers/${id}/events?limit=${limit}`); + if (!response.ok) { + throw new Error('Failed to fetch receiver events'); + } + return response.json(); + } +} + +export const receiversAPI = new ReceiversAPI(getAuthToken); diff --git a/packages/frontend/src/lib/components/receivers/CreateReceiverDialog.svelte b/packages/frontend/src/lib/components/receivers/CreateReceiverDialog.svelte new file mode 100644 index 00000000..7ef1f7ef --- /dev/null +++ b/packages/frontend/src/lib/components/receivers/CreateReceiverDialog.svelte @@ -0,0 +1,273 @@ + + + + + {#if !created} + + Create Webhook Receiver + + Receive events from an external system and store them as log entries in this project. + + + +
+
+ + +

+ Used as the default service name for generic events. +

+
+ +
+ +
+ {#each ADAPTER_OPTIONS as option (option.value)} + + {/each} +
+
+ + {#if adapterType === 'generic'} +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+

+ Unmapped fields fall back to sensible defaults; the full payload is always kept in metadata. +

+
+ {/if} + + {#if error} +
+ {error} +
+ {/if} + + + + + +
+ {:else} + + Receiver Created + Point the external system at this URL to start receiving events. + + +
+ + Important: Save this URL now + + The URL contains the receiver token and is the only credential the external system needs. + This is the only time it will be shown. If you lose it, delete the receiver and create a new one. + + + +
+ +
+
+
+ {ingestUrl} +
+
+ +
+
+ +
+

Usage Example:

+ 
curl -X POST {ingestUrl} \
+  -H "Content-Type: application/json" \
+  -d '{`{"message": "deploy finished"}`}'
+
+
+ + + + + {/if} + + diff --git a/packages/frontend/src/lib/components/receivers/ReceiverEventsDialog.svelte b/packages/frontend/src/lib/components/receivers/ReceiverEventsDialog.svelte new file mode 100644 index 00000000..2311efe0 --- /dev/null +++ b/packages/frontend/src/lib/components/receivers/ReceiverEventsDialog.svelte @@ -0,0 +1,128 @@ + + + + + + Recent Events{receiver ? ` - ${receiver.name}` : ''} + + The last events received by this webhook, with their normalized output. Only the most recent + 100 events are kept. + + + + {#if loading} +
+ + Loading events... +
+ {:else if events.length === 0} +
+

No events received yet

+

Send a test payload to the webhook URL to see it here.

+
+ {:else} +
+ {#each events as event (event.id)} +
+ + {#if expandedId === event.id} +
+
+

Raw Payload

+ 
{JSON.stringify(event.rawPayload, null, 2)}
+
+ {#if event.normalized} +
+

Normalized Logs

+ 
{JSON.stringify(event.normalized, null, 2)}
+
+ {/if} + {#if event.error} +
+

Error

+

{event.error}

+
+ {/if} +
+ {/if} +
+ {/each} +
+ {/if} + + diff --git a/packages/frontend/src/lib/components/receivers/index.ts b/packages/frontend/src/lib/components/receivers/index.ts new file mode 100644 index 00000000..278b4f5a --- /dev/null +++ b/packages/frontend/src/lib/components/receivers/index.ts @@ -0,0 +1,2 @@ +export { default as CreateReceiverDialog } from './CreateReceiverDialog.svelte'; +export { default as ReceiverEventsDialog } from './ReceiverEventsDialog.svelte'; diff --git a/packages/frontend/src/routes/dashboard/projects/[id]/settings/+page.svelte b/packages/frontend/src/routes/dashboard/projects/[id]/settings/+page.svelte index 65650590..0f8db77a 100644 --- a/packages/frontend/src/routes/dashboard/projects/[id]/settings/+page.svelte +++ b/packages/frontend/src/routes/dashboard/projects/[id]/settings/+page.svelte @@ -18,7 +18,10 @@ import Spinner from '$lib/components/Spinner.svelte'; import * as AlertDialog from '$lib/components/ui/alert-dialog'; import CreateApiKeyDialog from '$lib/components/CreateApiKeyDialog.svelte'; - import { Trash2, Plus } from '@lucide/svelte/icons'; + import { CreateReceiverDialog, ReceiverEventsDialog } from '$lib/components/receivers'; + import Switch from '$lib/components/ui/switch/switch.svelte'; + import { receiversAPI, type Receiver, type ReceiverAdapterType, type ReceiverFieldMapping } from '$lib/api/receivers'; + import { Trash2, Plus, ScrollText } from '@lucide/svelte/icons'; let project = $state(null); let loading = $state(false); @@ -36,6 +39,14 @@ let apiKeyToDelete = $state(null); let lastLoadedApiKeysKey = $state(null); + let receivers = $state([]); + let loadingReceivers = $state(false); + let showCreateReceiverDialog = $state(false); + let receiverToDelete = $state(null); + let eventsReceiver = $state(null); + let showEventsDialog = $state(false); + let lastLoadedReceiversKey = $state(null); + const projectId = $derived(page.params.id); async function loadProject() { @@ -92,6 +103,18 @@ loadApiKeys(); }); + $effect(() => { + if (!browser || !projectId) { + receivers = []; + lastLoadedReceiversKey = null; + return; + } + + if (projectId === lastLoadedReceiversKey) return; + + loadReceivers(); + }); + async function handleSave() { if (!$currentOrganization || !projectId) return; @@ -169,6 +192,73 @@ } } + async function loadReceivers() { + if (!projectId) return; + + loadingReceivers = true; + + try { + const response = await receiversAPI.list(projectId); + receivers = response.receivers; + lastLoadedReceiversKey = projectId; + } catch (e) { + toastStore.error(e instanceof Error ? e.message : 'Failed to load receivers'); + } finally { + loadingReceivers = false; + } + } + + async function handleCreateReceiver(data: { + name: string; + adapterType: ReceiverAdapterType; + fieldMapping?: ReceiverFieldMapping | null; + }) { + if (!projectId) throw new Error('No project ID'); + + const response = await receiversAPI.create(projectId, data); + toastStore.success('Receiver created successfully'); + await loadReceivers(); + return response; + } + + async function handleToggleReceiver(receiver: Receiver, enabled: boolean) { + if (!projectId) return; + + try { + await receiversAPI.update(projectId, receiver.id, { enabled }); + toastStore.success(enabled ? 'Receiver enabled' : 'Receiver disabled'); + await loadReceivers(); + } catch (e) { + toastStore.error(e instanceof Error ? e.message : 'Failed to update receiver'); + await loadReceivers(); + } + } + + async function handleDeleteReceiver() { + if (!projectId || !receiverToDelete) return; + + try { + await receiversAPI.delete(projectId, receiverToDelete.id); + toastStore.success('Receiver deleted successfully'); + await loadReceivers(); + } catch (e) { + toastStore.error(e instanceof Error ? e.message : 'Failed to delete receiver'); + } finally { + receiverToDelete = null; + } + } + + function openReceiverEvents(receiver: Receiver) { + eventsReceiver = receiver; + showEventsDialog = true; + } + + function adapterLabel(type: ReceiverAdapterType): string { + if (type === 'github') return 'GitHub'; + if (type === 'uptime') return 'Uptime'; + return 'Generic JSON'; + } + function formatDate(date: string | null): string { if (!date) return 'Never'; return new Date(date).toLocaleString('en-US'); @@ -360,6 +450,91 @@ + + +
+
+ Webhook Receivers + + Receive events from external systems (CI/CD, uptime monitors) as log entries + +
+ +
+ + + {#if loadingReceivers} +
+ + Loading receivers... +
+ {:else if receivers.length === 0} +
+

No receivers yet

+

Create a receiver to ingest events from external tools

+
+ {:else} +
+ + + + + + + + + + + + {#each receivers as receiver (receiver.id)} + + + + + + + + {/each} + +
NameSourceEnabledLast ReceivedActions
{receiver.name} + + {adapterLabel(receiver.adapterType)} + + + handleToggleReceiver(receiver, checked)} + /> + + {formatDate(receiver.lastReceivedAt)} + + + +
+
+ {/if} + + + Danger Zone @@ -384,6 +559,30 @@ + + + + + !open && (receiverToDelete = null)} +> + + + Delete Receiver + + Are you sure you want to delete the receiver {receiverToDelete?.name}? + Its webhook URL will stop working immediately and cannot be restored. This action cannot + be undone. + + + + Cancel + Delete Receiver + + + + !open && (apiKeyToDelete = null)} From 20e8be97057df1857c396d79ea5a0c68c0742176 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:25:46 +0200 Subject: [PATCH 13/23] document webhook receivers --- CHANGELOG.md | 1 + docs/receivers.md | 122 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 docs/receivers.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 416153dc..79031d70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Inbound webhook receivers** (#155): external systems (CI/CD, uptime monitors, arbitrary tools) can now push events into LogTide through per-receiver tokenized endpoints (`POST /api/v1/receivers/:id/:token`, `lr_`-prefixed tokens stored as SHA-256 hashes, timing-safe compare; the URL is the credential since GitHub/Uptime Robot cannot send custom headers). Three adapters normalize payloads into log entries: GitHub (workflow_run and deployment_status events with conclusion-to-level mapping; ping and unsupported events marked skipped), Uptime (shape-detects Uptime Robot alertType and Better Stack incident payloads), and Generic JSON (optional dot-path field mapping with levelMap and defaults, validated by a shared Zod schema; full payload always preserved in metadata). The endpoint answers 202 and processing is asynchronous via a new `receiver-events` queue job (queue abstraction, both backends), which runs the adapter, validates against `logSchema` and ingests through `ingestionService.ingestLogs`, so PII masking (fail-closed), usage quotas, Sigma detection, pipelines, metering and live tail all apply automatically; the outcome (processed/skipped/failed, normalized output, error) is recorded on a `receiver_events` row pruned to the last 100 per receiver. Management: project-scoped CRUD + recent-events API under `/api/v1/projects/:projectId/receivers` (session auth, audit-logged as `receiver.created`/`receiver.deleted`), a new `receivers.max` capability limit enforced with the canonical withLimitLock pattern (4-case + race tests), and a "Webhook Receivers" section in project settings (create dialog with adapter picker and generic field-mapping inputs, one-time URL reveal with copy, enable/disable switch, recent-events viewer with raw/normalized JSON). Migration 052 (`receivers`, `receiver_events`); public docs in `docs/receivers.md` - **Soft-delete for projects with a 30-day grace window**: deleting a project now moves it to a recoverable "trash" state (`deleted_at`) instead of removing it immediately. Its logs, traces and metrics stay queryable and the project stays viewable read-only during the window (the project detail layout loads with `includeDeleted`, shows a "deleted (read-only)" banner with the permanent-deletion date, and hides the Settings tab); a "Restore" action brings it back. A daily purge worker (3 AM) hard-deletes projects past the grace window, calling `reservoir.purgeProject()` to clear logs/spans/metrics from every storage engine before deleting the row. Name and slug uniqueness moved to partial unique indexes (active projects only) so a deleted project's name/slug can be reused; restoring into a name or slug a new active project has since taken is rejected with a 409 instead of a raw constraint error. Soft-deleted projects are excluded from listings, data-availability widgets and API-key verification (ingestion is refused immediately, with the key-verification cache invalidated on delete). New `POST /api/v1/projects/:id/restore` and `GET /api/v1/projects?includeDeleted=true`. Migrations 050 (soft-delete column + partial indexes) and 051 (drop `ON DELETE CASCADE` from logs/spans/metrics so an out-of-band project delete cannot bulk-wipe data; the purge worker clears reservoir data explicitly first) ## [1.0.3] - 2026-06-26 diff --git a/docs/receivers.md b/docs/receivers.md new file mode 100644 index 00000000..dacf0efe --- /dev/null +++ b/docs/receivers.md @@ -0,0 +1,122 @@ +# Webhook Receivers + +Webhook receivers let external systems (CI/CD pipelines, uptime monitors, arbitrary tools) push events into LogTide. Each receiver exposes a unique tokenized URL; incoming payloads are normalized into standard log entries by an adapter and stored through the regular ingestion pipeline, so PII masking, usage quotas, Sigma detection and parsing rules all apply automatically. + +Receivers are project-scoped and managed from **Project Settings > Webhook Receivers**, where you can create receivers, copy their URL, enable or disable them, and inspect the most recent received events with their normalized output. + +## Endpoint contract + +``` +POST /api/v1/receivers/{receiverId}/{token} +Content-Type: application/json +``` + +The URL itself is the credential: `receiverId` identifies the receiver and `token` authenticates the request. No headers are required, which makes the endpoint compatible with systems that cannot send custom headers (GitHub webhooks, Uptime Robot). + +The body must be a single JSON object of at most 256 KB. + +### Responses + +| Status | Meaning | +| --- | --- | +| `202 Accepted` | Event stored and queued for processing. Body: `{ "eventId": "..." }` | +| `400 Bad Request` | Body is not a JSON object (arrays, scalars and null are rejected) | +| `401 Unauthorized` | Token does not match the receiver | +| `403 Forbidden` | Receiver is disabled | +| `404 Not Found` | Unknown receiver id | +| `413 Payload Too Large` | Serialized payload exceeds 256 KB | + +Processing is asynchronous: a `202` means the payload was accepted, not that it produced a log entry. The outcome (`processed`, `skipped` or `failed`, with the normalized output and any error) is visible in the receiver's recent-events view, which keeps the last 100 events per receiver. + +## Tokens + +Receiver tokens use the `lr_` prefix (distinct from `lp_` project API keys) and are shown exactly once, at creation time, embedded in the full webhook URL. Only a SHA-256 hash is stored. Treat the URL as a secret: anyone who has it can write logs into your project. To rotate a token, delete the receiver and create a new one, then update the external system with the new URL. + +A receiver token cannot read data or call any other API endpoint; a leaked URL limits the blast radius to log ingestion into one project. + +## Adapters + +The adapter chosen at creation time decides how raw payloads become log entries. + +### GitHub + +Point a GitHub repository webhook (JSON content type) at the receiver URL. Supported events: + +- **workflow_run** (only `action: completed`): `success` maps to `info`, `cancelled`/`skipped`/`neutral` to `warn`, `failure`/`timed_out`/`startup_failure`/`action_required` to `error`. The message looks like `Workflow CI completed: failure`. +- **deployment_status**: `success` maps to `info`, `failure`/`error` to `error`, anything else to `info`. The message looks like `Deployment to production: failure`. + +The `service` field is the repository `full_name` (e.g. `acme/app`). Metadata includes the event type, workflow name, conclusion, run id and URL, branch and actor. Ping events and unsupported event types are marked `skipped`, not failed. + +Example: a failed CI run + +```json +{ + "action": "completed", + "workflow_run": { "name": "CI", "conclusion": "failure", "html_url": "...", "head_branch": "main" }, + "repository": { "full_name": "acme/app" }, + "sender": { "login": "octocat" } +} +``` + +becomes an `error` log for service `acme/app` with message `Workflow CI completed: failure`. + +### Uptime + +Detects two payload shapes automatically: + +- **Uptime Robot** (`alertType` field): `1` (down) maps to `error`, `2` (up) to `info`, `3` (SSL expiry) to `warn`. The monitor friendly name becomes the service. +- **Better Stack** (`data.attributes` incident shape): status `Started` maps to `error`, `Resolved` and `Acknowledged` to `info`. The monitor name becomes the service and the incident cause is part of the message. + +Unrecognized shapes are marked `skipped`. + +### Generic JSON + +Accepts any JSON object. Without configuration it produces one `info` log with message `Received event`, the receiver name as service, and the full payload preserved under `metadata.payload`. + +An optional **field mapping** extracts log fields from the payload using dot-paths: + +```json +{ + "message": "error.message", + "level": "severity", + "service": "source.app", + "timestamp": "ts", + "levelMap": { "crit": "critical", "sev1": "error" }, + "defaults": { "level": "warn", "service": "external-system" } +} +``` + +| Key | Meaning | +| --- | --- | +| `message` | Dot-path to the log message. Missing or non-string values fall back to `Received event`. | +| `level` | Dot-path to the level value. Values are lowercased and matched against LogTide levels (`debug`, `info`, `warn`, `error`, `critical`); the synonyms `warning`, `fatal`, `err` and `crit` are also understood. | +| `service` | Dot-path to the service name (truncated to 100 chars). Falls back to `defaults.service`, then the receiver name. | +| `timestamp` | Dot-path to an ISO string or epoch value. Unparseable values fall back to the arrival time. | +| `levelMap` | Case-insensitive map applied to the extracted level value before the builtin matching (e.g. map `sev1` to `error`). | +| `defaults` | Fallback `level` and `service` used when the mapped values are missing or invalid. | + +The create dialog exposes the four path fields; `levelMap` and `defaults` can be set through the API (`PATCH /api/v1/projects/{projectId}/receivers/{id}` with a `fieldMapping` object). + +## Pipeline guarantees + +Received events are ingested through the same path as `POST /api/v1/ingest`: + +- **PII masking is fail-closed**: entries whose masking fails are rejected, and the event is marked `failed` with the rejection reason. +- **Usage quotas** (`ingestion.max_bytes_monthly`, `ingestion.max_events_monthly`, `storage.max_bytes`) apply; over-quota events fail with the quota error recorded. +- Sigma detection, exception parsing, log pipelines, metering and live tail all see receiver events like any other ingested log. + +The number of receivers per organization can be capped with the `receivers.max` capability (unlimited by default in OSS). + +## Management API + +All management endpoints require session authentication and project access. + +| Method and path | Purpose | +| --- | --- | +| `GET /api/v1/projects/{projectId}/receivers` | List receivers (never returns tokens) | +| `POST /api/v1/projects/{projectId}/receivers` | Create; returns `token` and `ingestPath` once | +| `PATCH /api/v1/projects/{projectId}/receivers/{id}` | Rename, enable/disable, update `fieldMapping` | +| `DELETE /api/v1/projects/{projectId}/receivers/{id}` | Delete (invalidates the URL immediately) | +| `GET /api/v1/projects/{projectId}/receivers/{id}/events?limit=50` | Recent events with raw payload, normalized output and status | + +Receiver creation and deletion are recorded in the audit log (`receiver.created`, `receiver.deleted`). From 7d8cb99311f57275c77f30f7bb1853f67eb0f996 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:27:32 +0200 Subject: [PATCH 14/23] annotate receiver touch update for scoping tripwire --- packages/backend/src/modules/receivers/service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/modules/receivers/service.ts b/packages/backend/src/modules/receivers/service.ts index a46bcc14..e2653751 100644 --- a/packages/backend/src/modules/receivers/service.ts +++ b/packages/backend/src/modules/receivers/service.ts @@ -268,6 +268,7 @@ export class ReceiversService { } async touchLastReceived(receiverId: string): Promise { + // tenant-scope-ok: receiverId comes from a token-authenticated ingest request or a receiver row already loaded with its project join await db .updateTable('receivers') .set({ last_received_at: new Date() }) From c512d3cee4b10e0e2b55ceedea79d57d52db6431 Mon Sep 17 00:00:00 2001 From: Polliog Date: Thu, 2 Jul 2026 09:42:43 +0200 Subject: [PATCH 15/23] add receiver tables to table manifest test --- packages/backend/src/tests/database/tenant-tables.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/tests/database/tenant-tables.test.ts b/packages/backend/src/tests/database/tenant-tables.test.ts index 1fd5f6b3..65805348 100644 --- a/packages/backend/src/tests/database/tenant-tables.test.ts +++ b/packages/backend/src/tests/database/tenant-tables.test.ts @@ -18,7 +18,7 @@ const ALL_TABLES = [ 'organization_default_channels', 'pii_masking_rules', 'organization_pii_salts', 'audit_log', 'metrics_hourly_stats', 'metrics_daily_stats', 'metrics', 'metric_exemplars', 'custom_dashboards', 'log_pipelines', 'digest_configs', - 'digest_recipients', + 'digest_recipients', 'receivers', 'receiver_events', ]; describe('tenant-tables manifest', () => { From 2aa25bf705dced0c5e72fa959c96efe20bcf32e4 Mon Sep 17 00:00:00 2001 From: Polliog Date: Mon, 6 Jul 2026 17:14:58 +0200 Subject: [PATCH 16/23] use exact count below planner estimate threshold --- .../timescale/timescale-engine.test.ts | 36 +++++++++++++++++++ .../src/engines/timescale/timescale-engine.ts | 15 ++++++++ 2 files changed, 51 insertions(+) diff --git a/packages/reservoir/src/engines/timescale/timescale-engine.test.ts b/packages/reservoir/src/engines/timescale/timescale-engine.test.ts index 6af15de6..b540e381 100644 --- a/packages/reservoir/src/engines/timescale/timescale-engine.test.ts +++ b/packages/reservoir/src/engines/timescale/timescale-engine.test.ts @@ -369,6 +369,42 @@ describe('TimescaleEngine', () => { }); }); + describe('countEstimate', () => { + const countParams = { + projectId: 'proj-1', + from: new Date('2024-01-01'), + to: new Date('2024-01-02'), + }; + + it('returns an exact COUNT when the planner estimate is small', async () => { + // EXPLAIN reports a small estimate, so we fall back to an exact count. + mockQuery.mockResolvedValueOnce({ rows: [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': 3 } }] }] }); + mockQuery.mockResolvedValueOnce({ rows: [{ count: '42' }] }); + await engine.connect(); + + const result = await engine.countEstimate(countParams); + + expect(result.count).toBe(42); + expect(mockQuery).toHaveBeenCalledTimes(2); + const explainSql = mockQuery.mock.calls[0][0] as string; + const countSql = mockQuery.mock.calls[1][0] as string; + expect(explainSql).toContain('EXPLAIN (FORMAT JSON)'); + expect(countSql).toContain('COUNT(*)'); + expect(countSql).not.toContain('EXPLAIN'); + }); + + it('keeps the fast planner estimate for large datasets', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': 1_000_000 } }] }] }); + await engine.connect(); + + const result = await engine.countEstimate(countParams); + + expect(result.count).toBe(1_000_000); + // Only the EXPLAIN runs; no exact count. + expect(mockQuery).toHaveBeenCalledTimes(1); + }); + }); + describe('aggregate', () => { it('groups results by time bucket and level', async () => { const bucket1 = new Date('2024-01-01T00:00:00Z'); diff --git a/packages/reservoir/src/engines/timescale/timescale-engine.ts b/packages/reservoir/src/engines/timescale/timescale-engine.ts index b6130a53..b0172220 100644 --- a/packages/reservoir/src/engines/timescale/timescale-engine.ts +++ b/packages/reservoir/src/engines/timescale/timescale-engine.ts @@ -66,6 +66,11 @@ import { TimescaleQueryTranslator } from './query-translator.js'; const { Pool } = pg; +// Below this planner estimate we run an exact COUNT instead of trusting the estimate. +// At small scale an exact count is cheap and the planner estimate is unreliable; above +// it we keep the fast estimate so huge datasets stay performant. +const EXACT_COUNT_THRESHOLD = 50_000; + function sanitizeNull(value: string): string { return value.includes('\0') ? value.replace(/\0/g, '') : value; } @@ -379,6 +384,16 @@ export class TimescaleEngine extends StorageEngine { ); const plan = (result.rows[0] as Record)['QUERY PLAN'] as Array<{ Plan: { 'Plan Rows': number } }>; const estimate = Math.round(plan[0]?.Plan?.['Plan Rows'] ?? 0); + + // The planner estimate is unreliable at small scale (stale ANALYZE stats, few rows + // per chunk): it can be wrong in both directions, which users notice as a displayed + // total that disagrees with the visible logs. Below a threshold an exact COUNT is + // cheap, so return it; above it keep the fast estimate for large datasets. + if (estimate < EXACT_COUNT_THRESHOLD) { + const exact = await this.count(params); + return { count: exact.count, executionTimeMs: Date.now() - start }; + } + return { count: estimate, executionTimeMs: Date.now() - start, From 752985b8bb01c89b963db67933d2342e849d672b Mon Sep 17 00:00:00 2001 From: Polliog Date: Mon, 6 Jul 2026 17:14:58 +0200 Subject: [PATCH 17/23] fix dashboard panel uuid in non-secure http context --- CHANGELOG.md | 4 +++ .../custom-dashboards/panel-registry.ts | 3 ++- packages/frontend/src/lib/utils/uuid.test.ts | 27 +++++++++++++++++++ packages/frontend/src/lib/utils/uuid.ts | 22 +++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 packages/frontend/src/lib/utils/uuid.test.ts create mode 100644 packages/frontend/src/lib/utils/uuid.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 79031d70..c9bc46b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Inbound webhook receivers** (#155): external systems (CI/CD, uptime monitors, arbitrary tools) can now push events into LogTide through per-receiver tokenized endpoints (`POST /api/v1/receivers/:id/:token`, `lr_`-prefixed tokens stored as SHA-256 hashes, timing-safe compare; the URL is the credential since GitHub/Uptime Robot cannot send custom headers). Three adapters normalize payloads into log entries: GitHub (workflow_run and deployment_status events with conclusion-to-level mapping; ping and unsupported events marked skipped), Uptime (shape-detects Uptime Robot alertType and Better Stack incident payloads), and Generic JSON (optional dot-path field mapping with levelMap and defaults, validated by a shared Zod schema; full payload always preserved in metadata). The endpoint answers 202 and processing is asynchronous via a new `receiver-events` queue job (queue abstraction, both backends), which runs the adapter, validates against `logSchema` and ingests through `ingestionService.ingestLogs`, so PII masking (fail-closed), usage quotas, Sigma detection, pipelines, metering and live tail all apply automatically; the outcome (processed/skipped/failed, normalized output, error) is recorded on a `receiver_events` row pruned to the last 100 per receiver. Management: project-scoped CRUD + recent-events API under `/api/v1/projects/:projectId/receivers` (session auth, audit-logged as `receiver.created`/`receiver.deleted`), a new `receivers.max` capability limit enforced with the canonical withLimitLock pattern (4-case + race tests), and a "Webhook Receivers" section in project settings (create dialog with adapter picker and generic field-mapping inputs, one-time URL reveal with copy, enable/disable switch, recent-events viewer with raw/normalized JSON). Migration 052 (`receivers`, `receiver_events`); public docs in `docs/receivers.md` - **Soft-delete for projects with a 30-day grace window**: deleting a project now moves it to a recoverable "trash" state (`deleted_at`) instead of removing it immediately. Its logs, traces and metrics stay queryable and the project stays viewable read-only during the window (the project detail layout loads with `includeDeleted`, shows a "deleted (read-only)" banner with the permanent-deletion date, and hides the Settings tab); a "Restore" action brings it back. A daily purge worker (3 AM) hard-deletes projects past the grace window, calling `reservoir.purgeProject()` to clear logs/spans/metrics from every storage engine before deleting the row. Name and slug uniqueness moved to partial unique indexes (active projects only) so a deleted project's name/slug can be reused; restoring into a name or slug a new active project has since taken is rejected with a 409 instead of a raw constraint error. Soft-deleted projects are excluded from listings, data-availability widgets and API-key verification (ingestion is refused immediately, with the key-verification cache invalidated on delete). New `POST /api/v1/projects/:id/restore` and `GET /api/v1/projects?includeDeleted=true`. Migrations 050 (soft-delete column + partial indexes) and 051 (drop `ON DELETE CASCADE` from logs/spans/metrics so an out-of-band project delete cannot bulk-wipe data; the purge worker clears reservoir data explicitly first) +### Fixed +- **Log total no longer disagrees with the visible logs on TimescaleDB** (#271): the displayed result count came from the Postgres query planner's row estimate, which is unreliable at small scale (stale `ANALYZE` stats, few rows per chunk) and could be wrong in both directions, so the header showed a total that did not match the listed logs. `countEstimate` now falls back to an exact `COUNT` below a 50k-row estimate and keeps the fast planner estimate above it, so small and moderate deployments see an accurate total while large datasets stay performant. +- **Dashboard editing over plain HTTP on a LAN IP** (#272): adding a panel called `crypto.randomUUID()`, which only exists in a secure context (HTTPS or localhost), so it threw when LogTide was accessed over plain HTTP by IP. Panel IDs are now generated through a `uuid()` helper that falls back to `crypto.getRandomValues()` (available in non-secure contexts) when `randomUUID` is missing. + ## [1.0.3] - 2026-06-26 A security-focused release. It resolves a batch of privately reported issues (coordinated disclosure via KIberblick.de): cross-tenant read on the dashboard API endpoints, stored XSS via OTLP `service.name` in the service map, an open redirect on the auth-free login/register path, a DNS-rebinding gap in the SSRF guard's HTTP path, a first-admin bootstrap promotion race, and a capability-limit check-then-act race; trace span attributes are now PII-masked as well, and `service.name` is sanitized at ingestion as defense in depth. No database migrations; drop-in upgrade. Alongside the security work: two correctness follow-ups from the multi-engine bug-hunt sweep (issue #255): Sigma detection now honors full SigmaHQ field-modifier chains, and the service-map p95 is a true window percentile on every storage engine. The storage-layer change was validated against real ClickHouse, MongoDB and TimescaleDB. This line also fixes two operational bugs: a Redis memory leak where completed/failed BullMQ jobs were never evicted, and a nightly SigmaHQ sync that re-imported the whole catalog as enabled and auto-created alert rules. Plus a few frontend touch-ups: theme-aware trace/session IDs in the log detail, per-occurrence trace links on the error page, metadata copy buttons, a breadcrumbs timeline and nested metadata columns in log search. diff --git a/packages/frontend/src/lib/components/custom-dashboards/panel-registry.ts b/packages/frontend/src/lib/components/custom-dashboards/panel-registry.ts index 241bd514..df8e888b 100644 --- a/packages/frontend/src/lib/components/custom-dashboards/panel-registry.ts +++ b/packages/frontend/src/lib/components/custom-dashboards/panel-registry.ts @@ -51,6 +51,7 @@ import type { SystemStatusConfig, ActivityOverviewConfig, } from '@logtide/shared'; +import { uuid } from '$lib/utils/uuid'; import TimeSeriesPanel from './panels/TimeSeriesPanel.svelte'; import SingleStatPanel from './panels/SingleStatPanel.svelte'; @@ -385,7 +386,7 @@ export function getAllPanelDefinitions(): FrontendPanelDefinition[] { export function createPanelInstance(type: PanelType): PanelInstance { const def = getPanelDefinition(type); return { - id: `panel-${crypto.randomUUID()}`, + id: `panel-${uuid()}`, layout: { ...def.defaultLayout }, config: { ...def.defaultConfig }, }; diff --git a/packages/frontend/src/lib/utils/uuid.test.ts b/packages/frontend/src/lib/utils/uuid.test.ts new file mode 100644 index 00000000..5b34b90a --- /dev/null +++ b/packages/frontend/src/lib/utils/uuid.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { uuid } from './uuid'; + +const V4_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +describe('uuid', () => { + const originalRandomUUID = crypto.randomUUID; + + afterEach(() => { + crypto.randomUUID = originalRandomUUID; + }); + + it('produces a valid v4 UUID', () => { + expect(uuid()).toMatch(V4_RE); + }); + + it('produces unique values', () => { + const set = new Set(Array.from({ length: 1000 }, () => uuid())); + expect(set.size).toBe(1000); + }); + + it('falls back to getRandomValues when randomUUID is unavailable (non-secure context)', () => { + // Simulate plain HTTP on a LAN IP where crypto.randomUUID does not exist. + (crypto as { randomUUID?: unknown }).randomUUID = undefined; + expect(uuid()).toMatch(V4_RE); + }); +}); diff --git a/packages/frontend/src/lib/utils/uuid.ts b/packages/frontend/src/lib/utils/uuid.ts new file mode 100644 index 00000000..559a776d --- /dev/null +++ b/packages/frontend/src/lib/utils/uuid.ts @@ -0,0 +1,22 @@ +/** + * Generate an RFC 4122 v4 UUID. + * + * crypto.randomUUID() only exists in a secure context (HTTPS or localhost), so it + * throws when LogTide is accessed over plain HTTP on a LAN IP. crypto.getRandomValues() + * is available in non-secure contexts too, so we build the UUID from it as a fallback. + */ +export function uuid(): string { + if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { + return crypto.randomUUID(); + } + + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + + // Per RFC 4122 section 4.4: set the version (4) and variant (10xx) bits. + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')); + return `${hex.slice(0, 4).join('')}-${hex.slice(4, 6).join('')}-${hex.slice(6, 8).join('')}-${hex.slice(8, 10).join('')}-${hex.slice(10, 16).join('')}`; +} From e7f8b26f32e2752ac1e43755b7454e77602a249e Mon Sep 17 00:00:00 2001 From: Polliog Date: Tue, 7 Jul 2026 19:43:04 +0200 Subject: [PATCH 18/23] honor requested window for hostname dropdown --- CHANGELOG.md | 1 + packages/backend/src/modules/query/service.ts | 15 +++++++------- .../tests/modules/query/query-service.test.ts | 20 +++++++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9bc46b4..487b7f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Log total no longer disagrees with the visible logs on TimescaleDB** (#271): the displayed result count came from the Postgres query planner's row estimate, which is unreliable at small scale (stale `ANALYZE` stats, few rows per chunk) and could be wrong in both directions, so the header showed a total that did not match the listed logs. `countEstimate` now falls back to an exact `COUNT` below a 50k-row estimate and keeps the fast planner estimate above it, so small and moderate deployments see an accurate total while large datasets stay performant. - **Dashboard editing over plain HTTP on a LAN IP** (#272): adding a panel called `crypto.randomUUID()`, which only exists in a secure context (HTTPS or localhost), so it threw when LogTide was accessed over plain HTTP by IP. Panel IDs are now generated through a `uuid()` helper that falls back to `crypto.getRandomValues()` (available in non-secure contexts) when `randomUUID` is missing. +- **Hosts missing from the filter dropdown** (#273): `getDistinctHostnames` clamped its lookup window to 6 hours regardless of the selected range, so hosts that had only logged 6-24h ago were dropped from the hosts dropdown even though their logs were still visible in the (24h) logs view. The window now honors the requested range (default 24h, matching the log query default) so the dropdown lists every host with logs in the same window; hostnames stay cached for 5 minutes to bound the JSONB distinct cost. ## [1.0.3] - 2026-06-26 diff --git a/packages/backend/src/modules/query/service.ts b/packages/backend/src/modules/query/service.ts index 8ca77872..78ba850b 100644 --- a/packages/backend/src/modules/query/service.ts +++ b/packages/backend/src/modules/query/service.ts @@ -484,20 +484,19 @@ export class QueryService { * Hostnames are extracted from metadata.hostname field. * Cached for performance - used for filter dropdowns. * - * PERFORMANCE: Window is capped at 6 hours regardless of what the caller passes. - * JSONB extraction is expensive on large datasets - 6h ≈ 350ms, 24h+ ≈ 8s+. - * For a filter dropdown this is an acceptable trade-off: hostnames are stable. - * With 5-minute cache, most requests are served from cache after the first hit. + * The window honors what the caller passes (default 24h, matching the log query + * default) so the dropdown lists every host that has logs in the same range the + * logs view shows. Previously it was clamped to 6h, which silently dropped hosts + * that only logged 6-24h ago even though their logs were still visible. JSONB + * distinct is O(rows), but hostnames are stable and the 5-minute cache means the + * cost is paid at most once per window per cache period. */ async getDistinctHostnames( projectId: string | string[], from?: Date, to?: Date ): Promise { - // PERFORMANCE: Cap window to 6h max. If the caller requests a longer window - // (e.g. 24h), silently clamp it - JSONB distinct over large ranges is O(rows). - const sixHoursAgo = new Date(Date.now() - 6 * 60 * 60 * 1000); - const effectiveFrom = !from || from < sixHoursAgo ? sixHoursAgo : from; + const effectiveFrom = from ?? new Date(Date.now() - 24 * 60 * 60 * 1000); // Try cache first const cacheKey = CacheManager.statsKey( diff --git a/packages/backend/src/tests/modules/query/query-service.test.ts b/packages/backend/src/tests/modules/query/query-service.test.ts index f10ab253..a762ae13 100644 --- a/packages/backend/src/tests/modules/query/query-service.test.ts +++ b/packages/backend/src/tests/modules/query/query-service.test.ts @@ -512,6 +512,26 @@ describe('QueryService', () => { expect(result).toContain('server1.example.com'); expect(result).toContain('server2.example.com'); }); + + it('should include hosts that only logged 6-24h ago (not clamped to 6h)', async () => { + // Regression for #273: the lookup window was clamped to 6h, so a host that + // last logged 10h ago was missing from the dropdown while its logs stayed + // visible in the (24h) logs view. + const tenHoursAgo = new Date(Date.now() - 10 * 60 * 60 * 1000); + + await db.insertInto('logs').values({ + project_id: projectId, + service: 'api', + level: 'info', + message: 'Test', + time: tenHoursAgo, + metadata: { hostname: 'old-host.example.com' }, + }).execute(); + + const result = await queryService.getDistinctHostnames(projectId); + + expect(result).toContain('old-host.example.com'); + }); }); describe('getTopErrors', () => { From 839059ad774ff30913f4b63e06ffc855fb9718db Mon Sep 17 00:00:00 2001 From: Polliog Date: Wed, 8 Jul 2026 00:41:14 +0200 Subject: [PATCH 19/23] backfill activity overview recent tail from raw --- CHANGELOG.md | 1 + .../custom-dashboards/panel-data-service.ts | 167 +++++++++++------- .../panel-cagg-window.test.ts | 49 +++++ 3 files changed, 155 insertions(+), 62 deletions(-) create mode 100644 packages/backend/src/tests/modules/custom-dashboards/panel-cagg-window.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 487b7f49..99576cf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Log total no longer disagrees with the visible logs on TimescaleDB** (#271): the displayed result count came from the Postgres query planner's row estimate, which is unreliable at small scale (stale `ANALYZE` stats, few rows per chunk) and could be wrong in both directions, so the header showed a total that did not match the listed logs. `countEstimate` now falls back to an exact `COUNT` below a 50k-row estimate and keeps the fast planner estimate above it, so small and moderate deployments see an accurate total while large datasets stay performant. - **Dashboard editing over plain HTTP on a LAN IP** (#272): adding a panel called `crypto.randomUUID()`, which only exists in a secure context (HTTPS or localhost), so it threw when LogTide was accessed over plain HTTP by IP. Panel IDs are now generated through a `uuid()` helper that falls back to `crypto.getRandomValues()` (available in non-secure contexts) when `randomUUID` is missing. - **Hosts missing from the filter dropdown** (#273): `getDistinctHostnames` clamped its lookup window to 6 hours regardless of the selected range, so hosts that had only logged 6-24h ago were dropped from the hosts dropdown even though their logs were still visible in the (24h) logs view. The window now honors the requested range (default 24h, matching the log query default) so the dropdown lists every host with logs in the same window; hostnames stay cached for 5 minutes to bound the JSONB distinct cost. +- **Activity Overview chart stuck 1-2h behind live data** (#274): on TimescaleDB the panel read its logs/spans/detections series entirely from continuous aggregates, whose refresh policy (`end_offset = 1 bucket`, running once per bucket) never materializes the most recent 1-2 buckets, so the chart's latest edge showed empty while logs were still arriving (the raw-backed logs view stayed current, which read like a timezone offset). The panel now reads the cagg only up to a bucket-aligned cutoff two buckets back and backfills the recent tail live from the raw tables, keeping the fast aggregate for the bulk of the window while the latest buckets track real time. ClickHouse/MongoDB are unaffected (no continuous aggregates); alerts already read raw. ## [1.0.3] - 2026-06-26 diff --git a/packages/backend/src/modules/custom-dashboards/panel-data-service.ts b/packages/backend/src/modules/custom-dashboards/panel-data-service.ts index 612e5f7a..6b0f894e 100644 --- a/packages/backend/src/modules/custom-dashboards/panel-data-service.ts +++ b/packages/backend/src/modules/custom-dashboards/panel-data-service.ts @@ -655,10 +655,16 @@ const activityOverviewFetcher: PanelDataSource< const needsLogs = enabled.has('logs') || enabled.has('log_errors'); const needsSpans = enabled.has('spans') || enabled.has('span_errors'); - // Strategy: read from continuous aggregates first (cheap). If an aggregate - // returns zero rows for the window we fall back to the raw hypertable so - // freshly-ingested data isn't hidden by the `end_offset=1 hour` lag on the - // caggs' refresh policy. Alerts have no cagg so they always hit raw. + // Continuous aggregates lag real time: their refresh policy has + // `end_offset = 1 bucket` and only runs every bucket, so the most recent 1-2 + // buckets are never materialized while the raw tables are already live. Reading + // the cagg for the whole window therefore leaves the chart's latest buckets stuck + // at zero even though logs are still arriving (#274). Strategy: read the cagg only + // up to `caggCutoff` and backfill the recent tail [caggCutoff, now] from the raw + // tables. The cutoff is bucket-aligned and set two buckets back from the current + // one, comfortably past the cagg's materialization edge, so the two reads are + // disjoint (no double counting). Alerts have no cagg so they always hit raw. + const { caggCutoff, tailEnd } = computeCaggBackfillWindow(now, bucket); const logsRawTrunc = bucket === 'hour' ? sql`date_trunc('hour', time)` @@ -681,6 +687,21 @@ const activityOverviewFetcher: PanelDataSource< if (needsLogs && projectIds.length > 0) { if (isTimescale) { const logsTable = bucket === 'hour' ? 'logs_hourly_stats' : 'logs_daily_stats'; + // Raw log counts grouped by bucket over a half-open [fromD, toD) window. + const rawLogs = (fromD: Date, toD: Date) => + db + .selectFrom('logs') + .select([ + logsRawTrunc.as('bucket'), + 'level', + sql`COUNT(*)`.as('count'), + ]) + .where('project_id', 'in', projectIds) + .where('time', '>=', fromD) + .where('time', '<', toD) + .groupBy([logsRawTrunc, 'level']) + .execute() + .catch(() => [] as Array<{ bucket: unknown; level: string; count: string }>); tasks.push( (async () => { const caggRows = await db @@ -692,28 +713,17 @@ const activityOverviewFetcher: PanelDataSource< ]) .where('project_id', 'in', projectIds) .where('bucket', '>=', from) - .where('bucket', '<=', now) + .where('bucket', '<', caggCutoff) .groupBy(['bucket', 'level']) .execute() .catch(() => [] as Array<{ bucket: unknown; level: string; count: string }>); - const rows = caggRows.length > 0 - ? caggRows - : await db - .selectFrom('logs') - .select([ - logsRawTrunc.as('bucket'), - 'level', - sql`COUNT(*)`.as('count'), - ]) - .where('project_id', 'in', projectIds) - .where('time', '>=', from) - .where('time', '<=', now) - .groupBy([logsRawTrunc, 'level']) - .execute() - .catch(() => [] as Array<{ bucket: unknown; level: string; count: string }>); + // Older part from the cagg (raw fallback when the cagg is still empty, + // e.g. a fresh install) plus the recent tail always read live from raw. + const olderRows = caggRows.length > 0 ? caggRows : await rawLogs(from, caggCutoff); + const tailRows = await rawLogs(caggCutoff, tailEnd); - for (const r of rows) { + for (const r of [...olderRows, ...tailRows]) { const key = new Date(r.bucket as unknown as string).toISOString(); const i = index.get(key); if (i === undefined) continue; @@ -760,6 +770,23 @@ const activityOverviewFetcher: PanelDataSource< // interface has no aggregateSpans() primitive, so we follow the same // convention as trace_latency / trace_volume. const spansTable = bucket === 'hour' ? 'spans_hourly_stats' : 'spans_daily_stats'; + // Raw span counts grouped by bucket over a half-open [fromD, toD) window. + const rawSpans = (fromD: Date, toD: Date) => + db + .selectFrom('spans') + .select([ + spansRawTrunc.as('bucket'), + sql`COUNT(*)`.as('total'), + sql`SUM(CASE WHEN status_code = 'ERROR' THEN 1 ELSE 0 END)`.as( + 'errors', + ), + ]) + .where('project_id', 'in', projectIds) + .where('start_time', '>=', fromD) + .where('start_time', '<', toD) + .groupBy(spansRawTrunc) + .execute() + .catch(() => [] as Array<{ bucket: unknown; total: string; errors: string }>); tasks.push( (async () => { const caggRows = await db @@ -771,30 +798,15 @@ const activityOverviewFetcher: PanelDataSource< ]) .where('project_id', 'in', projectIds) .where('bucket', '>=', from) - .where('bucket', '<=', now) + .where('bucket', '<', caggCutoff) .groupBy('bucket') .execute() .catch(() => [] as Array<{ bucket: unknown; total: string; errors: string }>); - const rows = caggRows.length > 0 - ? caggRows - : await db - .selectFrom('spans') - .select([ - spansRawTrunc.as('bucket'), - sql`COUNT(*)`.as('total'), - sql`SUM(CASE WHEN status_code = 'ERROR' THEN 1 ELSE 0 END)`.as( - 'errors', - ), - ]) - .where('project_id', 'in', projectIds) - .where('start_time', '>=', from) - .where('start_time', '<=', now) - .groupBy(spansRawTrunc) - .execute() - .catch(() => [] as Array<{ bucket: unknown; total: string; errors: string }>); - - for (const r of rows) { + const olderRows = caggRows.length > 0 ? caggRows : await rawSpans(from, caggCutoff); + const tailRows = await rawSpans(caggCutoff, tailEnd); + + for (const r of [...olderRows, ...tailRows]) { const key = new Date(r.bucket as unknown as string).toISOString(); const i = index.get(key); if (i === undefined) continue; @@ -810,6 +822,25 @@ const activityOverviewFetcher: PanelDataSource< bucket === 'hour' ? 'detection_events_hourly_stats' : 'detection_events_daily_stats'; + // Raw detection counts grouped by bucket over a half-open [fromD, toD) window. + const rawDetections = (fromD: Date, toD: Date) => { + let rawQuery = db + .selectFrom('detection_events') + .select([ + detectionsRawTrunc.as('bucket'), + sql`COUNT(*)`.as('count'), + ]) + .where('organization_id', '=', ctx.organizationId) + .where('time', '>=', fromD) + .where('time', '<', toD); + if (config.projectId) { + rawQuery = rawQuery.where('project_id', '=', config.projectId); + } + return rawQuery + .groupBy(detectionsRawTrunc) + .execute() + .catch(() => [] as Array<{ bucket: unknown; count: string }>); + }; tasks.push( (async () => { let caggQuery = db @@ -820,7 +851,7 @@ const activityOverviewFetcher: PanelDataSource< ]) .where('organization_id', '=', ctx.organizationId) .where('bucket', '>=', from) - .where('bucket', '<=', now); + .where('bucket', '<', caggCutoff); if (config.projectId) { caggQuery = caggQuery.where('project_id', '=', config.projectId); } @@ -829,27 +860,10 @@ const activityOverviewFetcher: PanelDataSource< .execute() .catch(() => [] as Array<{ bucket: unknown; count: string }>); - let rows: Array<{ bucket: unknown; count: string }> = caggRows; - if (rows.length === 0) { - let rawQuery = db - .selectFrom('detection_events') - .select([ - detectionsRawTrunc.as('bucket'), - sql`COUNT(*)`.as('count'), - ]) - .where('organization_id', '=', ctx.organizationId) - .where('time', '>=', from) - .where('time', '<=', now); - if (config.projectId) { - rawQuery = rawQuery.where('project_id', '=', config.projectId); - } - rows = await rawQuery - .groupBy(detectionsRawTrunc) - .execute() - .catch(() => [] as Array<{ bucket: unknown; count: string }>); - } + const olderRows = caggRows.length > 0 ? caggRows : await rawDetections(from, caggCutoff); + const tailRows = await rawDetections(caggCutoff, tailEnd); - for (const r of rows) { + for (const r of [...olderRows, ...tailRows]) { const key = new Date(r.bucket as unknown as string).toISOString(); const i = index.get(key); if (i === undefined) continue; @@ -1054,6 +1068,35 @@ async function ensureProjectInOrg( } } +/** + * Window split for the Activity Overview cagg backfill (#274). + * + * Continuous aggregates lag real time (refresh policy `end_offset = 1 bucket`, + * running once per bucket), so the most recent 1-2 buckets are never materialized. + * `caggCutoff` is the bucket-aligned boundary two buckets back from the current + * (partial) bucket: read the cagg strictly before it and backfill [caggCutoff, now] + * live from the raw tables. Two buckets back sits comfortably past the cagg's + * materialization edge, so the cagg and raw reads never overlap (no double counting). + * `tailEnd` is an exclusive upper bound one millisecond past `now` so a `time < tailEnd` + * predicate still includes rows stamped exactly at `now`. + */ +export function computeCaggBackfillWindow( + now: Date, + bucket: 'hour' | 'day', +): { caggCutoff: Date; tailEnd: Date } { + const stepMs = bucket === 'hour' ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000; + const nowBucketStart = new Date(now); + if (bucket === 'hour') { + nowBucketStart.setUTCMinutes(0, 0, 0); + } else { + nowBucketStart.setUTCHours(0, 0, 0, 0); + } + return { + caggCutoff: new Date(nowBucketStart.getTime() - 2 * stepMs), + tailEnd: new Date(now.getTime() + 1), + }; +} + function buildBucketTimes( from: Date, to: Date, diff --git a/packages/backend/src/tests/modules/custom-dashboards/panel-cagg-window.test.ts b/packages/backend/src/tests/modules/custom-dashboards/panel-cagg-window.test.ts new file mode 100644 index 00000000..c1acee9a --- /dev/null +++ b/packages/backend/src/tests/modules/custom-dashboards/panel-cagg-window.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from 'vitest'; +import { computeCaggBackfillWindow } from '../../../modules/custom-dashboards/panel-data-service.js'; + +// Regression coverage for #274: the Activity Overview panel read continuous +// aggregates for the whole window, so the most recent 1-2 (unmaterialized) buckets +// showed empty while logs were still arriving. The fix reads the cagg up to +// `caggCutoff` and backfills [caggCutoff, now] from the raw tables. These tests pin +// the boundary math that keeps the two reads disjoint and the tail inclusive of now. +describe('computeCaggBackfillWindow', () => { + it('cuts the cagg off two whole hours before the current hour bucket', () => { + const now = new Date('2026-07-07T15:52:34.123Z'); + const { caggCutoff, tailEnd } = computeCaggBackfillWindow(now, 'hour'); + + expect(caggCutoff.toISOString()).toBe('2026-07-07T13:00:00.000Z'); + // tailEnd is exclusive but one ms past now, so `time < tailEnd` still includes now. + expect(tailEnd.getTime()).toBe(now.getTime() + 1); + expect(tailEnd.getTime()).toBeGreaterThan(now.getTime()); + }); + + it('cuts the cagg off two whole days before the current day bucket', () => { + const now = new Date('2026-07-07T15:52:34.123Z'); + const { caggCutoff, tailEnd } = computeCaggBackfillWindow(now, 'day'); + + expect(caggCutoff.toISOString()).toBe('2026-07-05T00:00:00.000Z'); + expect(tailEnd.getTime()).toBe(now.getTime() + 1); + }); + + it('aligns the cutoff to the bucket boundary (zeroed sub-bucket fields)', () => { + const now = new Date('2026-07-07T15:52:34.123Z'); + + const hourCutoff = computeCaggBackfillWindow(now, 'hour').caggCutoff; + expect(hourCutoff.getUTCMinutes()).toBe(0); + expect(hourCutoff.getUTCSeconds()).toBe(0); + expect(hourCutoff.getUTCMilliseconds()).toBe(0); + + const dayCutoff = computeCaggBackfillWindow(now, 'day').caggCutoff; + expect(dayCutoff.getUTCHours()).toBe(0); + expect(dayCutoff.getUTCMinutes()).toBe(0); + }); + + it('keeps the cutoff strictly before now so the raw tail is non-empty', () => { + const now = new Date('2026-07-07T15:00:00.000Z'); // exactly on a bucket boundary + const { caggCutoff } = computeCaggBackfillWindow(now, 'hour'); + + // Even when now sits on a boundary, the cutoff is two full buckets back. + expect(caggCutoff.toISOString()).toBe('2026-07-07T13:00:00.000Z'); + expect(caggCutoff.getTime()).toBeLessThan(now.getTime()); + }); +}); From 576c47f7d58ca2ef3b51fe2d5f1c8567eaae1754 Mon Sep 17 00:00:00 2001 From: Polliog Date: Fri, 10 Jul 2026 14:00:39 +0200 Subject: [PATCH 20/23] label top services widget with its 7-day window --- CHANGELOG.md | 1 + .../src/lib/components/dashboard/TopServicesWidget.svelte | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99576cf0..b7e38a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Dashboard editing over plain HTTP on a LAN IP** (#272): adding a panel called `crypto.randomUUID()`, which only exists in a secure context (HTTPS or localhost), so it threw when LogTide was accessed over plain HTTP by IP. Panel IDs are now generated through a `uuid()` helper that falls back to `crypto.getRandomValues()` (available in non-secure contexts) when `randomUUID` is missing. - **Hosts missing from the filter dropdown** (#273): `getDistinctHostnames` clamped its lookup window to 6 hours regardless of the selected range, so hosts that had only logged 6-24h ago were dropped from the hosts dropdown even though their logs were still visible in the (24h) logs view. The window now honors the requested range (default 24h, matching the log query default) so the dropdown lists every host with logs in the same window; hostnames stay cached for 5 minutes to bound the JSONB distinct cost. - **Activity Overview chart stuck 1-2h behind live data** (#274): on TimescaleDB the panel read its logs/spans/detections series entirely from continuous aggregates, whose refresh policy (`end_offset = 1 bucket`, running once per bucket) never materializes the most recent 1-2 buckets, so the chart's latest edge showed empty while logs were still arriving (the raw-backed logs view stayed current, which read like a timezone offset). The panel now reads the cagg only up to a bucket-aligned cutoff two buckets back and backfills the recent tail live from the raw tables, keeping the fast aggregate for the bulk of the window while the latest buckets track real time. ClickHouse/MongoDB are unaffected (no continuous aggregates); alerts already read raw. +- **Top Services table window was unlabeled** (#275): the "Top Services by Volume" widget covers a fixed 7-day window (and its total is a 7-day total), but it sat next to the 24h-framed activity chart with no window label, so its service list and total looked inconsistent with the logs page (which follows the selected range, 24h by default) and read like stale/stuck data. The widget title now states its window ("Last 7 Days") so the different counts are self-explanatory; the 7-day window itself is intentional (daily continuous aggregate for the historical span) and unchanged. ## [1.0.3] - 2026-06-26 diff --git a/packages/frontend/src/lib/components/dashboard/TopServicesWidget.svelte b/packages/frontend/src/lib/components/dashboard/TopServicesWidget.svelte index 09f32821..58f1cb95 100644 --- a/packages/frontend/src/lib/components/dashboard/TopServicesWidget.svelte +++ b/packages/frontend/src/lib/components/dashboard/TopServicesWidget.svelte @@ -18,7 +18,7 @@ - Top Services by Volume + Top Services by Volume (Last 7 Days)
From f7993d01231172d5fbdfe139238a55557f837f08 Mon Sep 17 00:00:00 2001 From: Polliog Date: Fri, 10 Jul 2026 14:04:31 +0200 Subject: [PATCH 21/23] cut 1.1.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e38a15..f9007e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.1.0] - 2026-07-08 ### Added - **Inbound webhook receivers** (#155): external systems (CI/CD, uptime monitors, arbitrary tools) can now push events into LogTide through per-receiver tokenized endpoints (`POST /api/v1/receivers/:id/:token`, `lr_`-prefixed tokens stored as SHA-256 hashes, timing-safe compare; the URL is the credential since GitHub/Uptime Robot cannot send custom headers). Three adapters normalize payloads into log entries: GitHub (workflow_run and deployment_status events with conclusion-to-level mapping; ping and unsupported events marked skipped), Uptime (shape-detects Uptime Robot alertType and Better Stack incident payloads), and Generic JSON (optional dot-path field mapping with levelMap and defaults, validated by a shared Zod schema; full payload always preserved in metadata). The endpoint answers 202 and processing is asynchronous via a new `receiver-events` queue job (queue abstraction, both backends), which runs the adapter, validates against `logSchema` and ingests through `ingestionService.ingestLogs`, so PII masking (fail-closed), usage quotas, Sigma detection, pipelines, metering and live tail all apply automatically; the outcome (processed/skipped/failed, normalized output, error) is recorded on a `receiver_events` row pruned to the last 100 per receiver. Management: project-scoped CRUD + recent-events API under `/api/v1/projects/:projectId/receivers` (session auth, audit-logged as `receiver.created`/`receiver.deleted`), a new `receivers.max` capability limit enforced with the canonical withLimitLock pattern (4-case + race tests), and a "Webhook Receivers" section in project settings (create dialog with adapter picker and generic field-mapping inputs, one-time URL reveal with copy, enable/disable switch, recent-events viewer with raw/normalized JSON). Migration 052 (`receivers`, `receiver_events`); public docs in `docs/receivers.md` From 2a9a8e5e1ad430c741cc7f361ef20ea6f44a74ea Mon Sep 17 00:00:00 2001 From: Polliog Date: Fri, 10 Jul 2026 14:06:10 +0200 Subject: [PATCH 22/23] bump version to 1.1.0 --- package.json | 2 +- packages/backend/package.json | 2 +- packages/frontend/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 8748606e..b56f0384 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "logtide", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "LogTide - Self-hosted log management platform", "author": "LogTide Team", diff --git a/packages/backend/package.json b/packages/backend/package.json index ad30355a..5124cd30 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "@logtide/backend", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "LogTide Backend API", "type": "module", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index a2e3f1de..fd933557 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@logtide/frontend", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "LogTide Frontend Dashboard", "type": "module", From ceb4e816d8697daf51a63c9fccaecbe45023d3db Mon Sep 17 00:00:00 2001 From: Polliog Date: Fri, 10 Jul 2026 14:08:51 +0200 Subject: [PATCH 23/23] bump remaining version references to 1.1.0 --- README.md | 6 +++--- packages/backend/src/utils/internal-logger.ts | 2 +- packages/frontend/src/hooks.client.ts | 2 +- packages/frontend/src/hooks.server.ts | 2 +- packages/frontend/src/lib/components/Footer.svelte | 2 +- packages/reservoir/package.json | 2 +- packages/shared/package.json | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 35290eb9..286512d1 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,14 @@ Coverage Docker Artifact Hub - Version + Version License Status

-> **🌊 LogTide 1.0.3 (public beta):** unified **Logs, Traces & Metrics** with a built-in **SIEM**, multi-engine storage (TimescaleDB / ClickHouse / MongoDB), uptime monitoring, parsing pipelines, and custom dashboards. +> **🌊 LogTide 1.1.0 (public beta):** unified **Logs, Traces & Metrics** with a built-in **SIEM**, multi-engine storage (TimescaleDB / ClickHouse / MongoDB), uptime monitoring, parsing pipelines, and custom dashboards. --- @@ -124,7 +124,7 @@ We host it for you. Perfect for testing. [**Sign up at logtide.dev**](https://lo --- -## ✨ Core Features (v1.0.3) +## ✨ Core Features (v1.1.0) ### Monitoring, Pipelines & Dashboards * 🩺 **Uptime Monitoring & Status Pages:** HTTP/TCP/heartbeat monitors with configurable thresholds, auto-created SIEM incidents on failure, scheduled maintenances, and public Uptime-Kuma-style status pages per project. diff --git a/packages/backend/src/utils/internal-logger.ts b/packages/backend/src/utils/internal-logger.ts index 63886fcf..59d62287 100644 --- a/packages/backend/src/utils/internal-logger.ts +++ b/packages/backend/src/utils/internal-logger.ts @@ -58,7 +58,7 @@ export async function initializeInternalLogging(): Promise { dsn, service: process.env.SERVICE_NAME || 'logtide-backend', environment: process.env.NODE_ENV || 'development', - release: process.env.npm_package_version || '1.0.3', + release: process.env.npm_package_version || '1.1.0', batchSize: 5, // Smaller batch for internal logs to see them faster flushInterval: 5000, maxBufferSize: 1000, diff --git a/packages/frontend/src/hooks.client.ts b/packages/frontend/src/hooks.client.ts index b2fb82e1..d5e59755 100644 --- a/packages/frontend/src/hooks.client.ts +++ b/packages/frontend/src/hooks.client.ts @@ -9,7 +9,7 @@ if (dsn) { dsn, service: 'logtide-frontend-client', environment: env.PUBLIC_NODE_ENV || 'production', - release: env.PUBLIC_APP_VERSION || '1.0.3', + release: env.PUBLIC_APP_VERSION || '1.1.0', debug: env.PUBLIC_NODE_ENV === 'development', browser: { // Core Web Vitals (LCP, INP, CLS, TTFB) diff --git a/packages/frontend/src/hooks.server.ts b/packages/frontend/src/hooks.server.ts index 86bb0559..b208ad34 100644 --- a/packages/frontend/src/hooks.server.ts +++ b/packages/frontend/src/hooks.server.ts @@ -82,7 +82,7 @@ export const handle = dsn dsn, service: 'logtide-frontend', environment: privateEnv?.NODE_ENV || 'production', - release: process.env.npm_package_version || '1.0.3', }) as unknown as Handle, + release: process.env.npm_package_version || '1.1.0', }) as unknown as Handle, requestLogHandle, configHandle ) diff --git a/packages/frontend/src/lib/components/Footer.svelte b/packages/frontend/src/lib/components/Footer.svelte index 2df9517a..38c5aadc 100644 --- a/packages/frontend/src/lib/components/Footer.svelte +++ b/packages/frontend/src/lib/components/Footer.svelte @@ -1,7 +1,7 @@ diff --git a/packages/reservoir/package.json b/packages/reservoir/package.json index a7bb6d67..42556a3e 100644 --- a/packages/reservoir/package.json +++ b/packages/reservoir/package.json @@ -1,6 +1,6 @@ { "name": "@logtide/reservoir", - "version": "1.0.3", + "version": "1.1.0", "description": "Pluggable storage abstraction for Logtide log management", "type": "module", "main": "./dist/index.js", diff --git a/packages/shared/package.json b/packages/shared/package.json index 8f236971..f13e1e77 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@logtide/shared", - "version": "1.0.3", + "version": "1.1.0", "private": true, "description": "Shared types, schemas and utilities for LogTide", "type": "module",