From ea9bc77b36086b8c620e7d649401b571ce9aef39 Mon Sep 17 00:00:00 2001 From: Snider Date: Sat, 31 Jan 2026 20:03:22 +0000 Subject: [PATCH 1/3] ci: add dependabot configuration --- .github/dependabot.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..76f6a6e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,35 @@ +version: 2 +updates: + - package-ecosystem: "composer" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + labels: + - "type:dependencies" + - "priority:low" + commit-message: + prefix: "deps(composer):" + + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + labels: + - "type:dependencies" + - "priority:low" + commit-message: + prefix: "deps(npm):" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + labels: + - "type:dependencies" + - "priority:low" + commit-message: + prefix: "deps(actions):" + From 6153f6885cbcbc336b3e7dee4a9ee25efa0646da Mon Sep 17 00:00:00 2001 From: Snider Date: Sat, 31 Jan 2026 20:04:29 +0000 Subject: [PATCH 2/3] ci: add CodeQL security scanning --- .github/workflows/codeql.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..5ef77a8 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,36 @@ +name: CodeQL + +on: + push: + branches: [dev, main] + pull_request: + branches: [dev, main] + schedule: + - cron: "0 6 * * 1" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: javascript + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:javascript" + From 00b787a5681c02d7d8370ed400b96fe8bb2927ed Mon Sep 17 00:00:00 2001 From: Snider Date: Sat, 31 Jan 2026 20:58:31 +0000 Subject: [PATCH 3/3] docs: sync package documentation from all modules Synced documentation from 16 packages: - admin, api, commerce, content, developer, mcp, tenant (updated) - agentic, analytics, bio, notify, social, support, tools, trust, uptelligence (added) Co-Authored-By: Claude Opus 4.5 --- docs/packages/agentic/api-keys.md | 319 ++++++++ docs/packages/agentic/architecture.md | 322 ++++++++ docs/packages/agentic/mcp-tools.md | 670 ++++++++++++++++ docs/packages/agentic/security.md | 279 +++++++ docs/packages/analytics/api.md | 868 +++++++++++++++++++++ docs/packages/analytics/architecture.md | 419 ++++++++++ docs/packages/analytics/security.md | 461 +++++++++++ docs/packages/analytics/testing.md | 435 +++++++++++ docs/packages/bio/architecture.md | 396 ++++++++++ docs/packages/bio/block-types.md | 746 ++++++++++++++++++ docs/packages/bio/security.md | 438 +++++++++++ docs/packages/commerce/index.md | 34 - docs/packages/content/index.md | 39 - docs/packages/developer/index.md | 37 - docs/packages/notify/api.md | 541 +++++++++++++ docs/packages/notify/architecture.md | 332 ++++++++ docs/packages/notify/security.md | 252 ++++++ docs/packages/notify/webhooks.md | 395 ++++++++++ docs/packages/social/architecture.md | 503 ++++++++++++ docs/packages/social/providers.md | 528 +++++++++++++ docs/packages/social/security.md | 336 ++++++++ docs/packages/support/api.md | 545 +++++++++++++ docs/packages/support/architecture.md | 319 ++++++++ docs/packages/support/security.md | 289 +++++++ docs/packages/tenant/index.md | 48 -- docs/packages/tools/architecture.md | 384 +++++++++ docs/packages/tools/security.md | 279 +++++++ docs/packages/trust/architecture.md | 291 +++++++ docs/packages/trust/security.md | 268 +++++++ docs/packages/trust/widget-types.md | 576 ++++++++++++++ docs/packages/uptelligence/architecture.md | 345 ++++++++ docs/packages/uptelligence/security.md | 307 ++++++++ docs/packages/uptelligence/storage.md | 283 +++++++ docs/packages/uptelligence/webhooks.md | 306 ++++++++ 34 files changed, 12432 insertions(+), 158 deletions(-) create mode 100644 docs/packages/agentic/api-keys.md create mode 100644 docs/packages/agentic/architecture.md create mode 100644 docs/packages/agentic/mcp-tools.md create mode 100644 docs/packages/agentic/security.md create mode 100644 docs/packages/analytics/api.md create mode 100644 docs/packages/analytics/architecture.md create mode 100644 docs/packages/analytics/security.md create mode 100644 docs/packages/analytics/testing.md create mode 100644 docs/packages/bio/architecture.md create mode 100644 docs/packages/bio/block-types.md create mode 100644 docs/packages/bio/security.md delete mode 100644 docs/packages/commerce/index.md delete mode 100644 docs/packages/content/index.md delete mode 100644 docs/packages/developer/index.md create mode 100644 docs/packages/notify/api.md create mode 100644 docs/packages/notify/architecture.md create mode 100644 docs/packages/notify/security.md create mode 100644 docs/packages/notify/webhooks.md create mode 100644 docs/packages/social/architecture.md create mode 100644 docs/packages/social/providers.md create mode 100644 docs/packages/social/security.md create mode 100644 docs/packages/support/api.md create mode 100644 docs/packages/support/architecture.md create mode 100644 docs/packages/support/security.md delete mode 100644 docs/packages/tenant/index.md create mode 100644 docs/packages/tools/architecture.md create mode 100644 docs/packages/tools/security.md create mode 100644 docs/packages/trust/architecture.md create mode 100644 docs/packages/trust/security.md create mode 100644 docs/packages/trust/widget-types.md create mode 100644 docs/packages/uptelligence/architecture.md create mode 100644 docs/packages/uptelligence/security.md create mode 100644 docs/packages/uptelligence/storage.md create mode 100644 docs/packages/uptelligence/webhooks.md diff --git a/docs/packages/agentic/api-keys.md b/docs/packages/agentic/api-keys.md new file mode 100644 index 0000000..cb96e57 --- /dev/null +++ b/docs/packages/agentic/api-keys.md @@ -0,0 +1,319 @@ +--- +title: API Keys +description: Guide to Agent API key management +updated: 2026-01-29 +--- + +# API Key Management + +Agent API keys provide authenticated access to the MCP tools and agentic services. This guide covers key creation, permissions, and security. + +## Key Structure + +API keys follow the format: `ak_` + 32 random alphanumeric characters. + +Example: `ak_a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6` + +The key is only displayed once at creation. Store it securely. + +## Creating Keys + +### Via Admin Panel + +1. Navigate to Workspace Settings > API Keys +2. Click "Create New Key" +3. Enter a descriptive name +4. Select permissions +5. Set expiration (optional) +6. Click Create +7. Copy the displayed key immediately + +### Programmatically + +```php +use Core\Mod\Agentic\Services\AgentApiKeyService; + +$service = app(AgentApiKeyService::class); + +$key = $service->create( + workspace: $workspace, + name: 'My Agent Key', + permissions: [ + AgentApiKey::PERM_PLANS_READ, + AgentApiKey::PERM_PLANS_WRITE, + AgentApiKey::PERM_SESSIONS_WRITE, + ], + rateLimit: 100, + expiresAt: now()->addYear() +); + +// Only available once +$plainKey = $key->plainTextKey; +``` + +## Permissions + +### Available Permissions + +| Permission | Constant | Description | +|------------|----------|-------------| +| `plans.read` | `PERM_PLANS_READ` | List and view plans | +| `plans.write` | `PERM_PLANS_WRITE` | Create, update, archive plans | +| `phases.write` | `PERM_PHASES_WRITE` | Update phases, manage tasks | +| `sessions.read` | `PERM_SESSIONS_READ` | List and view sessions | +| `sessions.write` | `PERM_SESSIONS_WRITE` | Start, update, end sessions | +| `tools.read` | `PERM_TOOLS_READ` | View tool analytics | +| `templates.read` | `PERM_TEMPLATES_READ` | List and view templates | +| `templates.instantiate` | `PERM_TEMPLATES_INSTANTIATE` | Create plans from templates | +| `notify:read` | `PERM_NOTIFY_READ` | List push campaigns | +| `notify:write` | `PERM_NOTIFY_WRITE` | Create/update campaigns | +| `notify:send` | `PERM_NOTIFY_SEND` | Send notifications | + +### Permission Checking + +```php +// Single permission +$key->hasPermission('plans.write'); + +// Any of several +$key->hasAnyPermission(['plans.read', 'sessions.read']); + +// All required +$key->hasAllPermissions(['plans.write', 'phases.write']); +``` + +### Updating Permissions + +```php +$service->updatePermissions($key, [ + AgentApiKey::PERM_PLANS_READ, + AgentApiKey::PERM_SESSIONS_READ, +]); +``` + +## Rate Limiting + +### Configuration + +Each key has a configurable rate limit (requests per minute): + +```php +$key = $service->create( + workspace: $workspace, + name: 'Limited Key', + permissions: [...], + rateLimit: 50 // 50 requests/minute +); + +// Update later +$service->updateRateLimit($key, 100); +``` + +### Checking Status + +```php +$status = $service->getRateLimitStatus($key); +// Returns: +// [ +// 'limit' => 100, +// 'remaining' => 85, +// 'reset_in_seconds' => 45, +// 'used' => 15 +// ] +``` + +### Response Headers + +Rate limit info is included in API responses: + +``` +X-RateLimit-Limit: 100 +X-RateLimit-Remaining: 85 +X-RateLimit-Reset: 45 +``` + +When rate limited (HTTP 429): +``` +Retry-After: 45 +``` + +## IP Restrictions + +Keys can be restricted to specific IP addresses or ranges. + +### Enabling Restrictions + +```php +// Enable with whitelist +$service->enableIpRestrictions($key, [ + '192.168.1.0/24', // CIDR range + '10.0.0.5', // Single IPv4 + '2001:db8::1', // Single IPv6 + '2001:db8::/32', // IPv6 CIDR +]); + +// Disable restrictions +$service->disableIpRestrictions($key); +``` + +### Managing Whitelist + +```php +// Add single entry +$key->addToIpWhitelist('192.168.2.0/24'); + +// Remove entry +$key->removeFromIpWhitelist('192.168.1.0/24'); + +// Replace entire list +$key->updateIpWhitelist([ + '10.0.0.0/8', + '172.16.0.0/12', +]); +``` + +### Parsing Input + +For user-entered whitelists: + +```php +$result = $service->parseIpWhitelistInput(" + 192.168.1.1 + 192.168.2.0/24 + # This is a comment + invalid-ip +"); + +// Result: +// [ +// 'entries' => ['192.168.1.1', '192.168.2.0/24'], +// 'errors' => ['invalid-ip: Invalid IP address'] +// ] +``` + +## Key Lifecycle + +### Expiration + +```php +// Set expiration on create +$key = $service->create( + ... + expiresAt: now()->addMonths(6) +); + +// Extend expiration +$service->extendExpiry($key, now()->addYear()); + +// Remove expiration (never expires) +$service->removeExpiry($key); +``` + +### Revocation + +```php +// Immediately revoke +$service->revoke($key); + +// Check status +$key->isRevoked(); // true +$key->isActive(); // false +``` + +### Status Helpers + +```php +$key->isActive(); // Not revoked, not expired +$key->isRevoked(); // Has been revoked +$key->isExpired(); // Past expiration date +$key->getStatusLabel(); // "Active", "Revoked", or "Expired" +``` + +## Authentication + +### Making Requests + +Include the API key as a Bearer token: + +```bash +curl -H "Authorization: Bearer ak_your_key_here" \ + https://mcp.host.uk.com/api/agent/plans +``` + +### Authentication Flow + +1. Middleware extracts Bearer token +2. Key looked up by SHA-256 hash +3. Status checked (revoked, expired) +4. IP validated if restrictions enabled +5. Permissions checked against required scopes +6. Rate limit checked and incremented +7. Usage recorded (count, timestamp, IP) + +### Error Responses + +| HTTP Code | Error | Description | +|-----------|-------|-------------| +| 401 | `unauthorised` | Missing or invalid key | +| 401 | `key_revoked` | Key has been revoked | +| 401 | `key_expired` | Key has expired | +| 403 | `ip_not_allowed` | Request IP not whitelisted | +| 403 | `permission_denied` | Missing required permission | +| 429 | `rate_limited` | Rate limit exceeded | + +## Usage Tracking + +Each key tracks: +- `call_count` - Total lifetime calls +- `last_used_at` - Timestamp of last use +- `last_used_ip` - IP of last request + +Access via: +```php +$key->call_count; +$key->getLastUsedForHumans(); // "2 hours ago" +``` + +## Best Practices + +1. **Use descriptive names** - "Production Agent" not "Key 1" +2. **Minimal permissions** - Only grant needed scopes +3. **Set expiration** - Rotate keys periodically +4. **Enable IP restrictions** - When agents run from known IPs +5. **Monitor usage** - Review call patterns regularly +6. **Revoke promptly** - If key may be compromised +7. **Separate environments** - Different keys for dev/staging/prod + +## Example: Complete Setup + +```php +use Core\Mod\Agentic\Services\AgentApiKeyService; +use Core\Mod\Agentic\Models\AgentApiKey; + +$service = app(AgentApiKeyService::class); + +// Create a production key +$key = $service->create( + workspace: $workspace, + name: 'Production Agent - Claude', + permissions: [ + AgentApiKey::PERM_PLANS_READ, + AgentApiKey::PERM_PLANS_WRITE, + AgentApiKey::PERM_PHASES_WRITE, + AgentApiKey::PERM_SESSIONS_WRITE, + AgentApiKey::PERM_TEMPLATES_READ, + AgentApiKey::PERM_TEMPLATES_INSTANTIATE, + ], + rateLimit: 200, + expiresAt: now()->addYear() +); + +// Restrict to known IPs +$service->enableIpRestrictions($key, [ + '203.0.113.0/24', // Office network + '198.51.100.50', // CI/CD server +]); + +// Store the key securely +$plainKey = $key->plainTextKey; // Only chance to get this! +``` diff --git a/docs/packages/agentic/architecture.md b/docs/packages/agentic/architecture.md new file mode 100644 index 0000000..e393fed --- /dev/null +++ b/docs/packages/agentic/architecture.md @@ -0,0 +1,322 @@ +--- +title: Architecture +description: Technical architecture of the core-agentic package +updated: 2026-01-29 +--- + +# Architecture + +The `core-agentic` package provides AI agent orchestration infrastructure for the Host UK platform. It enables multi-agent collaboration, persistent task tracking, and unified access to multiple AI providers. + +## Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ MCP Protocol Layer │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ Plan │ │ Phase │ │ Session │ │ State │ ... tools │ +│ │ Tools │ │ Tools │ │ Tools │ │ Tools │ │ +│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ +└───────┼────────────┼────────────┼────────────┼──────────────────┘ + │ │ │ │ +┌───────┴────────────┴────────────┴────────────┴──────────────────┐ +│ AgentToolRegistry │ +│ - Tool registration and discovery │ +│ - Permission checking (API key scopes) │ +│ - Dependency validation │ +│ - Circuit breaker integration │ +└──────────────────────────────────────────────────────────────────┘ + │ +┌───────┴──────────────────────────────────────────────────────────┐ +│ Core Services │ +│ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ │ +│ │ AgenticManager │ │ AgentApiKey │ │ PlanTemplate │ │ +│ │ (AI Providers) │ │ Service │ │ Service │ │ +│ └────────────────┘ └────────────────┘ └────────────────┘ │ +│ ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ │ +│ │ IpRestriction │ │ Content │ │ AgentSession │ │ +│ │ Service │ │ Service │ │ Service │ │ +│ └────────────────┘ └────────────────┘ └────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ + │ +┌───────┴──────────────────────────────────────────────────────────┐ +│ Data Layer │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐│ +│ │ AgentPlan │ │ AgentPhase │ │ AgentSession│ │ AgentApiKey ││ +│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘│ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ Workspace │ │ Task │ │ +│ │ State │ │ │ │ +│ └─────────────┘ └─────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## Core Concepts + +### Agent Plans + +Plans represent structured work with phases, tasks, and progress tracking. They persist across agent sessions, enabling handoff between different AI models or instances. + +``` +AgentPlan +├── slug (unique identifier) +├── title +├── status (draft → active → completed/archived) +├── current_phase +└── phases[] (AgentPhase) + ├── name + ├── tasks[] + │ ├── name + │ └── status + ├── dependencies[] + └── checkpoints[] +``` + +**Lifecycle:** +1. Created via MCP tool or template +2. Activated to begin work +3. Phases started/completed in order +4. Plan auto-completes when all phases done +5. Archived for historical reference + +### Agent Sessions + +Sessions track individual work periods. They enable context recovery when an agent's context window resets or when handing off to another agent. + +``` +AgentSession +├── session_id (prefixed unique ID) +├── agent_type (opus/sonnet/haiku) +├── status (active/paused/completed/failed) +├── work_log[] (chronological actions) +├── artifacts[] (files created/modified) +├── context_summary (current state) +└── handoff_notes (for next agent) +``` + +**Handoff Flow:** +1. Session logs work as it progresses +2. Before context ends, agent calls `session_handoff` +3. Handoff notes capture summary, next steps, blockers +4. Next agent calls `session_resume` to continue +5. Resume session inherits context from previous + +### Workspace State + +Key-value state storage shared between sessions and plans. Enables agents to persist decisions, configurations, and intermediate results. + +``` +WorkspaceState +├── key (namespaced identifier) +├── value (any JSON-serialisable data) +├── type (json/markdown/code/reference) +└── category (for organisation) +``` + +## MCP Tool Architecture + +All MCP tools extend the `AgentTool` base class which provides: + +### Input Validation + +```php +protected function requireString(array $args, string $key, ?int $maxLength = null): string +protected function optionalInt(array $args, string $key, ?int $default = null): ?int +protected function requireEnum(array $args, string $key, array $allowed): string +``` + +### Circuit Breaker Protection + +```php +return $this->withCircuitBreaker('agentic', function () { + // Database operations that could fail + return AgentPlan::where('slug', $slug)->first(); +}, fn () => $this->error('Service unavailable', 'circuit_open')); +``` + +### Dependency Declaration + +```php +public function dependencies(): array +{ + return [ + ToolDependency::contextExists('workspace_id', 'Workspace required'), + ToolDependency::toolCalled('session_start', 'Start session first'), + ]; +} +``` + +### Tool Categories + +| Category | Tools | Purpose | +|----------|-------|---------| +| `plan` | plan_create, plan_get, plan_list, plan_update_status, plan_archive | Work plan management | +| `phase` | phase_get, phase_update_status, phase_add_checkpoint | Phase operations | +| `session` | session_start, session_end, session_log, session_handoff, session_resume, session_replay | Session tracking | +| `state` | state_get, state_set, state_list | Persistent state | +| `task` | task_update, task_toggle | Task completion | +| `template` | template_list, template_preview, template_create_plan | Plan templates | +| `content` | content_generate, content_batch_generate, content_brief_create | Content generation | + +## AI Provider Abstraction + +The `AgenticManager` provides unified access to multiple AI providers: + +```php +$ai = app(AgenticManager::class); + +// Use specific provider +$response = $ai->claude()->generate($system, $user); +$response = $ai->gemini()->generate($system, $user); +$response = $ai->openai()->generate($system, $user); + +// Use by name (for configuration-driven selection) +$response = $ai->provider('gemini')->generate($system, $user); +``` + +### Provider Interface + +All providers implement `AgenticProviderInterface`: + +```php +interface AgenticProviderInterface +{ + public function generate(string $systemPrompt, string $userPrompt, array $config = []): AgenticResponse; + public function stream(string $systemPrompt, string $userPrompt, array $config = []): Generator; + public function name(): string; + public function defaultModel(): string; + public function isAvailable(): bool; +} +``` + +### Response Object + +```php +class AgenticResponse +{ + public string $content; + public string $model; + public int $inputTokens; + public int $outputTokens; + public int $durationMs; + public ?string $stopReason; + public array $raw; + + public function estimateCost(): float; +} +``` + +## Authentication + +### API Key Flow + +``` +Request → AgentApiAuth Middleware → AgentApiKeyService::authenticate() + │ + ├── Validate key (SHA-256 hash lookup) + ├── Check revoked/expired + ├── Validate IP whitelist + ├── Check permissions + ├── Check rate limit + └── Record usage +``` + +### Permission Model + +```php +// Permission constants +AgentApiKey::PERM_PLANS_READ // 'plans.read' +AgentApiKey::PERM_PLANS_WRITE // 'plans.write' +AgentApiKey::PERM_SESSIONS_WRITE // 'sessions.write' +// etc. + +// Check permissions +$key->hasPermission('plans.write'); +$key->hasAllPermissions(['plans.read', 'sessions.write']); +``` + +### IP Restrictions + +API keys can optionally restrict access by IP: + +- Individual IPv4/IPv6 addresses +- CIDR notation (e.g., `192.168.1.0/24`) +- Mixed whitelist + +## Event-Driven Boot + +The module uses the Core framework's event-driven lazy loading: + +```php +class Boot extends ServiceProvider +{ + public static array $listens = [ + AdminPanelBooting::class => 'onAdminPanel', + ConsoleBooting::class => 'onConsole', + McpToolsRegistering::class => 'onMcpTools', + ]; +} +``` + +This ensures: +- Views only loaded when admin panel boots +- Commands only registered when console boots +- MCP tools only registered when MCP module initialises + +## Multi-Tenancy + +All data is workspace-scoped via the `BelongsToWorkspace` trait: + +- Queries auto-scoped to current workspace +- Creates auto-assign workspace_id +- Cross-tenant queries throw `MissingWorkspaceContextException` + +## File Structure + +``` +core-agentic/ +├── Boot.php # Service provider with event handlers +├── config.php # Module configuration +├── Migrations/ # Database schema +├── Models/ # Eloquent models +│ ├── AgentPlan.php +│ ├── AgentPhase.php +│ ├── AgentSession.php +│ ├── AgentApiKey.php +│ └── WorkspaceState.php +├── Services/ # Business logic +│ ├── AgenticManager.php # AI provider orchestration +│ ├── AgentApiKeyService.php # API key management +│ ├── IpRestrictionService.php +│ ├── PlanTemplateService.php +│ ├── ContentService.php +│ ├── ClaudeService.php +│ ├── GeminiService.php +│ └── OpenAIService.php +├── Mcp/ +│ ├── Tools/Agent/ # MCP tool implementations +│ │ ├── AgentTool.php # Base class +│ │ ├── Plan/ +│ │ ├── Phase/ +│ │ ├── Session/ +│ │ ├── State/ +│ │ └── ... +│ ├── Prompts/ # MCP prompt definitions +│ └── Servers/ # MCP server configurations +├── Middleware/ +│ └── AgentApiAuth.php # API authentication +├── Controllers/ +│ └── ForAgentsController.php # Agent discovery endpoint +├── View/ +│ ├── Blade/admin/ # Admin panel views +│ └── Modal/Admin/ # Livewire components +├── Jobs/ # Queue jobs +├── Console/Commands/ # Artisan commands +└── Tests/ # Pest test suites +``` + +## Dependencies + +- `host-uk/core` - Event system, base classes +- `host-uk/core-tenant` - Workspace, BelongsToWorkspace trait +- `host-uk/core-mcp` - MCP infrastructure, CircuitBreaker diff --git a/docs/packages/agentic/mcp-tools.md b/docs/packages/agentic/mcp-tools.md new file mode 100644 index 0000000..da12266 --- /dev/null +++ b/docs/packages/agentic/mcp-tools.md @@ -0,0 +1,670 @@ +--- +title: MCP Tools Reference +description: Complete reference for core-agentic MCP tools +updated: 2026-01-29 +--- + +# MCP Tools Reference + +This document provides a complete reference for all MCP tools in the `core-agentic` package. + +## Overview + +Tools are organised into categories: + +| Category | Description | Tools Count | +|----------|-------------|-------------| +| plan | Work plan management | 5 | +| phase | Phase operations | 3 | +| session | Session tracking | 8 | +| state | Persistent state | 3 | +| task | Task completion | 2 | +| template | Plan templates | 3 | +| content | Content generation | 6 | + +## Plan Tools + +### plan_create + +Create a new work plan with phases and tasks. + +**Scopes:** `write` + +**Input:** +```json +{ + "title": "string (required)", + "slug": "string (optional, auto-generated)", + "description": "string (optional)", + "context": "object (optional)", + "phases": [ + { + "name": "string", + "description": "string", + "tasks": ["string"] + } + ] +} +``` + +**Output:** +```json +{ + "success": true, + "plan": { + "slug": "my-plan-abc123", + "title": "My Plan", + "status": "draft", + "phases": 3 + } +} +``` + +**Dependencies:** workspace_id in context + +--- + +### plan_get + +Get a plan by slug with full details. + +**Scopes:** `read` + +**Input:** +```json +{ + "slug": "string (required)" +} +``` + +**Output:** +```json +{ + "success": true, + "plan": { + "slug": "my-plan", + "title": "My Plan", + "status": "active", + "progress": { + "total": 5, + "completed": 2, + "percentage": 40 + }, + "phases": [...] + } +} +``` + +--- + +### plan_list + +List plans with optional filtering. + +**Scopes:** `read` + +**Input:** +```json +{ + "status": "string (optional: draft|active|completed|archived)", + "limit": "integer (optional, default 20)" +} +``` + +**Output:** +```json +{ + "success": true, + "plans": [ + { + "slug": "plan-1", + "title": "Plan One", + "status": "active" + } + ], + "count": 1 +} +``` + +--- + +### plan_update_status + +Update a plan's status. + +**Scopes:** `write` + +**Input:** +```json +{ + "slug": "string (required)", + "status": "string (required: draft|active|completed|archived)" +} +``` + +--- + +### plan_archive + +Archive a plan with optional reason. + +**Scopes:** `write` + +**Input:** +```json +{ + "slug": "string (required)", + "reason": "string (optional)" +} +``` + +## Phase Tools + +### phase_get + +Get phase details by plan slug and phase order. + +**Scopes:** `read` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "phase_order": "integer (required)" +} +``` + +--- + +### phase_update_status + +Update a phase's status. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "phase_order": "integer (required)", + "status": "string (required: pending|in_progress|completed|blocked|skipped)", + "reason": "string (optional, for blocked/skipped)" +} +``` + +--- + +### phase_add_checkpoint + +Add a checkpoint note to a phase. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "phase_order": "integer (required)", + "note": "string (required)", + "context": "object (optional)" +} +``` + +## Session Tools + +### session_start + +Start a new agent session. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (optional)", + "agent_type": "string (required: opus|sonnet|haiku)", + "context": "object (optional)" +} +``` + +**Output:** +```json +{ + "success": true, + "session": { + "session_id": "ses_abc123xyz", + "agent_type": "opus", + "plan": "my-plan", + "status": "active" + } +} +``` + +--- + +### session_end + +End a session with status and summary. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)", + "status": "string (required: completed|failed)", + "summary": "string (optional)" +} +``` + +--- + +### session_log + +Add a work log entry to an active session. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)", + "message": "string (required)", + "type": "string (optional: info|warning|error|success|checkpoint)", + "data": "object (optional)" +} +``` + +--- + +### session_handoff + +Prepare session for handoff to another agent. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)", + "summary": "string (required)", + "next_steps": ["string"], + "blockers": ["string"], + "context_for_next": "object (optional)" +} +``` + +--- + +### session_resume + +Resume a paused session. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)" +} +``` + +**Output:** +```json +{ + "success": true, + "session": {...}, + "handoff_context": { + "summary": "Previous work summary", + "next_steps": ["Continue with..."], + "blockers": [] + } +} +``` + +--- + +### session_replay + +Get replay context for a session. + +**Scopes:** `read` + +**Input:** +```json +{ + "session_id": "string (required)" +} +``` + +**Output:** +```json +{ + "success": true, + "replay_context": { + "session_id": "ses_abc123", + "progress_summary": {...}, + "last_checkpoint": {...}, + "decisions": [...], + "errors": [...] + } +} +``` + +--- + +### session_continue + +Create a new session that continues from a previous one. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)", + "agent_type": "string (optional)" +} +``` + +--- + +### session_artifact + +Add an artifact (file) to a session. + +**Scopes:** `write` + +**Input:** +```json +{ + "session_id": "string (required)", + "path": "string (required)", + "action": "string (required: created|modified|deleted)", + "metadata": "object (optional)" +} +``` + +--- + +### session_list + +List sessions with optional filtering. + +**Scopes:** `read` + +**Input:** +```json +{ + "plan_slug": "string (optional)", + "status": "string (optional)", + "limit": "integer (optional)" +} +``` + +## State Tools + +### state_set + +Set a workspace state value. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "key": "string (required)", + "value": "any (required)", + "category": "string (optional)" +} +``` + +--- + +### state_get + +Get a workspace state value. + +**Scopes:** `read` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "key": "string (required)" +} +``` + +--- + +### state_list + +List all state for a plan. + +**Scopes:** `read` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "category": "string (optional)" +} +``` + +## Task Tools + +### task_update + +Update a task within a phase. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "phase_order": "integer (required)", + "task_identifier": "string|integer (required)", + "status": "string (optional: pending|completed)", + "notes": "string (optional)" +} +``` + +--- + +### task_toggle + +Toggle a task's completion status. + +**Scopes:** `write` + +**Input:** +```json +{ + "plan_slug": "string (required)", + "phase_order": "integer (required)", + "task_identifier": "string|integer (required)" +} +``` + +## Template Tools + +### template_list + +List available plan templates. + +**Scopes:** `read` + +**Output:** +```json +{ + "success": true, + "templates": [ + { + "slug": "feature-development", + "name": "Feature Development", + "description": "Standard feature workflow", + "phases_count": 5, + "variables": [ + { + "name": "FEATURE_NAME", + "required": true + } + ] + } + ] +} +``` + +--- + +### template_preview + +Preview a template with variable substitution. + +**Scopes:** `read` + +**Input:** +```json +{ + "slug": "string (required)", + "variables": { + "FEATURE_NAME": "Authentication" + } +} +``` + +--- + +### template_create_plan + +Create a plan from a template. + +**Scopes:** `write` + +**Input:** +```json +{ + "template_slug": "string (required)", + "variables": "object (required)", + "title": "string (optional, overrides template)", + "activate": "boolean (optional, default false)" +} +``` + +## Content Tools + +### content_generate + +Generate content using AI. + +**Scopes:** `write` + +**Input:** +```json +{ + "prompt": "string (required)", + "provider": "string (optional: claude|gemini|openai)", + "config": { + "temperature": 0.7, + "max_tokens": 4000 + } +} +``` + +--- + +### content_batch_generate + +Generate content for a batch specification. + +**Scopes:** `write` + +**Input:** +```json +{ + "batch_id": "string (required)", + "provider": "string (optional)", + "dry_run": "boolean (optional)" +} +``` + +--- + +### content_brief_create + +Create a content brief for later generation. + +**Scopes:** `write` + +--- + +### content_brief_get + +Get a content brief. + +**Scopes:** `read` + +--- + +### content_brief_list + +List content briefs. + +**Scopes:** `read` + +--- + +### content_status + +Get batch generation status. + +**Scopes:** `read` + +--- + +### content_usage_stats + +Get AI usage statistics. + +**Scopes:** `read` + +--- + +### content_from_plan + +Generate content based on plan context. + +**Scopes:** `write` + +## Error Responses + +All tools return errors in this format: + +```json +{ + "error": "Error message", + "code": "error_code" +} +``` + +Common error codes: +- `validation_error` - Invalid input +- `not_found` - Resource not found +- `permission_denied` - Insufficient permissions +- `rate_limited` - Rate limit exceeded +- `service_unavailable` - Circuit breaker open + +## Circuit Breaker + +Tools use circuit breaker protection for database calls. When the circuit opens: + +```json +{ + "error": "Agentic service temporarily unavailable", + "code": "service_unavailable" +} +``` + +The circuit resets after successful health checks. diff --git a/docs/packages/agentic/security.md b/docs/packages/agentic/security.md new file mode 100644 index 0000000..d5bf2ef --- /dev/null +++ b/docs/packages/agentic/security.md @@ -0,0 +1,279 @@ +--- +title: Security +description: Security considerations and audit notes for core-agentic +updated: 2026-01-29 +--- + +# Security Considerations + +This document outlines security considerations, known issues, and recommendations for the `core-agentic` package. + +## Authentication + +### API Key Security + +**Current Implementation:** +- Keys generated with `ak_` prefix + 32 random characters +- Stored as SHA-256 hash (no salt) +- Key only visible once at creation time +- Supports expiration dates +- Supports revocation + +**Known Issues:** + +1. **No salt in hash (SEC-001)** + - Risk: Rainbow table attacks possible against common key formats + - Mitigation: Keys are high-entropy (32 random chars), reducing practical risk + - Recommendation: Migrate to Argon2id with salt + +2. **Key prefix visible in hash display** + - The `getMaskedKey()` method shows first 6 chars of the hash, not the original key + - This is safe but potentially confusing for users + +**Recommendations:** +- Consider key rotation reminders +- Add key compromise detection (unusual usage patterns) +- Implement key versioning for smooth rotation + +### IP Whitelisting + +**Implementation:** +- Per-key IP restriction toggle +- Supports IPv4 and IPv6 +- Supports CIDR notation +- Logged when requests blocked + +**Validation:** +- Uses `filter_var()` with `FILTER_VALIDATE_IP` +- CIDR prefix validated against IP version limits (0-32 for IPv4, 0-128 for IPv6) +- Normalises IPs for consistent comparison + +**Edge Cases Handled:** +- Empty whitelist with restrictions enabled = deny all +- Invalid IPs/CIDRs rejected during configuration +- IP version mismatch (IPv4 vs IPv6) handled correctly + +## Authorisation + +### Multi-Tenancy + +**Workspace Scoping:** +- All models use `BelongsToWorkspace` trait +- Queries automatically scoped to current workspace context +- Missing workspace throws `MissingWorkspaceContextException` + +**Known Issues:** + +1. **StateSet tool lacks workspace validation (SEC-003)** + - Risk: Plan lookup by slug without workspace constraint + - Impact: Could allow cross-tenant state manipulation if slugs collide + - Fix: Add workspace_id check to plan query + +2. **Some tools have soft dependency on workspace** + - SessionStart marks workspace as optional if plan_slug provided + - Could theoretically allow workspace inference attacks + +### Permission Model + +**Scopes:** +- `plans.read` - List and view plans +- `plans.write` - Create, update, archive plans +- `phases.write` - Update phase status, manage tasks +- `sessions.read` - List and view sessions +- `sessions.write` - Start, update, complete sessions +- `tools.read` - View tool analytics +- `templates.read` - List and view templates +- `templates.instantiate` - Create plans from templates + +**Tool Scope Enforcement:** +- Each tool declares required scopes +- `AgentToolRegistry::execute()` validates scopes before execution +- Missing scope throws `RuntimeException` + +## Rate Limiting + +### Current Implementation + +**Global Rate Limiting:** +- ForAgentsController: 60 requests/minute per IP +- Configured via `RateLimiter::for('agentic-api')` + +**Per-Key Rate Limiting:** +- Configurable per API key (default: 100/minute) +- Uses cache-based counter with 60-second TTL +- Atomic increment via `Cache::add()` + `Cache::increment()` + +**Known Issues:** + +1. **No per-tool rate limiting (SEC-004)** + - Risk: Single key can call expensive tools unlimited times + - Impact: Resource exhaustion, cost overrun + - Fix: Add tool-specific rate limits + +2. **Rate limit counter not distributed** + - Multiple app servers may have separate counters + - Fix: Ensure Redis cache driver in production + +### Response Headers + +Rate limit status exposed via headers: +- `X-RateLimit-Limit` - Maximum requests allowed +- `X-RateLimit-Remaining` - Requests remaining in window +- `X-RateLimit-Reset` - Seconds until reset +- `Retry-After` - When rate limited + +## Input Validation + +### MCP Tool Inputs + +**Validation Helpers:** +- `requireString()` - Type + optional length validation +- `requireInt()` - Type + optional min/max validation +- `requireEnum()` - Value from allowed set +- `requireArray()` - Type validation + +**Known Issues:** + +1. **Template variable injection (VAL-001)** + - JSON escaping added but character validation missing + - Risk: Specially crafted variables could affect template behaviour + - Recommendation: Add explicit character whitelist + +2. **SQL orderByRaw pattern (SEC-002)** + - TaskCommand uses raw SQL for FIELD() ordering + - Currently safe (hardcoded values) but fragile pattern + - Recommendation: Use parameterised approach + +### Content Validation + +ContentService validates generated content: +- Minimum word count (600 words) +- UK English spelling checks +- Banned word detection +- Structure validation (headings required) + +## Data Protection + +### Sensitive Data Handling + +**API Keys:** +- Plaintext only available once (at creation) +- Hash stored, never logged +- Excluded from model serialisation via `$hidden` + +**Session Data:** +- Work logs may contain sensitive context +- Artifacts track file paths (not contents) +- Context summaries could contain user data + +**Recommendations:** +- Add data retention policies for sessions +- Consider encrypting context_summary field +- Audit work_log for sensitive data patterns + +### Logging + +**Current Logging:** +- IP restriction blocks logged with key metadata +- No API key plaintext ever logged +- No sensitive context logged + +**Recommendations:** +- Add audit logging for permission changes +- Log key creation/revocation events +- Consider structured logging for SIEM integration + +## Transport Security + +**Requirements:** +- All endpoints should be HTTPS-only +- MCP portal at mcp.host.uk.com +- API endpoints under /api/agent/* + +**Headers Set:** +- `X-Client-IP` - For debugging/audit +- Rate limit headers + +**Recommendations:** +- Add HSTS headers +- Consider mTLS for high-security deployments + +## Dependency Security + +### External API Calls + +AI provider services make external API calls: +- Anthropic API (Claude) +- Google AI API (Gemini) +- OpenAI API + +**Security Measures:** +- API keys from environment variables only +- HTTPS connections +- 300-second timeout +- Retry with exponential backoff + +**Recommendations:** +- Consider API key vault integration +- Add certificate pinning for provider endpoints +- Monitor for API key exposure in responses + +### Internal Dependencies + +The package depends on: +- `host-uk/core` - Event system +- `host-uk/core-tenant` - Workspace scoping +- `host-uk/core-mcp` - MCP infrastructure + +All are internal packages with shared security posture. + +## Audit Checklist + +### Pre-Production + +- [ ] All SEC-* issues in TODO.md addressed +- [ ] API key hashing upgraded to Argon2id +- [ ] StateSet workspace scoping fixed +- [ ] Per-tool rate limiting implemented +- [ ] Test coverage for auth/permission logic + +### Regular Audits + +- [ ] Review API key usage patterns +- [ ] Check for expired but not revoked keys +- [ ] Audit workspace scope bypass attempts +- [ ] Review rate limit effectiveness +- [ ] Check for unusual tool call patterns + +### Incident Response + +1. **Compromised API Key** + - Immediately revoke via `$key->revoke()` + - Check usage history in database + - Notify affected workspace owner + - Review all actions taken with key + +2. **Cross-Tenant Access** + - Disable affected workspace + - Audit all data access + - Review workspace scoping logic + - Implement additional checks + +## Security Contacts + +For security issues: +- Create private issue in repository +- Email security@host.uk.com +- Do not disclose publicly until patched + +## Changelog + +**2026-01-29** +- Initial security documentation +- Documented known issues SEC-001 through SEC-004 +- Added audit checklist + +**2026-01-21** +- Rate limiting functional (was stub) +- Admin routes now require Hades role +- ForAgentsController rate limited diff --git a/docs/packages/analytics/api.md b/docs/packages/analytics/api.md new file mode 100644 index 0000000..aa0435f --- /dev/null +++ b/docs/packages/analytics/api.md @@ -0,0 +1,868 @@ +--- +title: API Reference +description: REST API documentation for core-analytics +updated: 2026-01-29 +--- + +# API Reference + +This document describes the REST API for the core-analytics package. + +## Base URL + +All authenticated endpoints are prefixed with `/analytics`. + +## Authentication + +### Public Endpoints + +Tracking endpoints do not require authentication but need a valid `pixel_key`: + +``` +POST /analytics/track/{pixelKey} +POST /analytics/track +GET /analytics/pixel?key={pixelKey} +POST /analytics/heartbeat +POST /analytics/leave +POST /analytics/event +``` + +### Authenticated Endpoints + +Management endpoints require authentication via the Core PHP Framework auth system: + +``` +Authorization: Bearer {token} +``` + +## Rate Limiting + +Public tracking endpoints: 10,000 requests/minute (shared pool) + +## Tracking Endpoints + +### Track Pageview (POST) + +Track a pageview or event. + +**Endpoint:** `POST /analytics/track/{pixelKey}` + +**Response:** 1x1 transparent GIF (always returns success to avoid exposing errors) + +--- + +### Track Pageview (JSON) + +Track with full JSON payload. + +**Endpoint:** `POST /analytics/track` + +**Request Body:** +```json +{ + "key": "uuid-pixel-key", + "type": "pageview", + "visitor_id": "visitor-uuid", + "session_id": "session-uuid", + "path": "/page/path", + "title": "Page Title", + "referrer": "https://referrer.com", + "utm_source": "google", + "utm_medium": "cpc", + "utm_campaign": "summer-sale", + "screen_width": 1920, + "screen_height": 1080, + "language": "en-GB" +} +``` + +**Response:** +```json +{ + "ok": true, + "event_id": 12345, + "visitor_id": "visitor-uuid", + "session_id": "session-uuid" +} +``` + +--- + +### Track Pixel (GET) + +Lightweight tracking for noscript fallback. + +**Endpoint:** `GET /analytics/pixel?key={pixelKey}&p={path}&t={title}&r={referrer}` + +**Response:** 1x1 transparent GIF + +--- + +### Heartbeat + +Update time on page and scroll depth. + +**Endpoint:** `POST /analytics/heartbeat` + +**Request Body:** +```json +{ + "event_id": 12345, + "time_on_page": 120, + "scroll_depth": 75, + "session_id": "session-uuid" +} +``` + +**Response:** +```json +{ + "ok": true +} +``` + +--- + +### Leave + +End session on page unload (sendBeacon). + +**Endpoint:** `POST /analytics/leave` + +**Request Body:** +```json +{ + "session_id": "session-uuid" +} +``` + +--- + +### Custom Event + +Track a custom event. + +**Endpoint:** `POST /analytics/event` + +**Request Body:** +```json +{ + "key": "uuid-pixel-key", + "name": "button_click", + "visitor_id": "visitor-uuid", + "session_id": "session-uuid", + "properties": { + "button_id": "signup", + "variant": "blue" + } +} +``` + +--- + +## Website Management + +### List Websites + +**Endpoint:** `GET /analytics/websites` + +**Response:** +```json +{ + "data": [ + { + "id": 1, + "name": "My Website", + "host": "example.com", + "pixel_key": "uuid", + "tracking_enabled": true, + "is_enabled": true, + "created_at": "2026-01-01T00:00:00Z" + } + ] +} +``` + +--- + +### Create Website + +**Endpoint:** `POST /analytics/websites` + +**Request Body:** +```json +{ + "name": "My Website", + "host": "example.com", + "tracking_type": "lightweight", + "channel_type": "website" +} +``` + +--- + +### Get Website + +**Endpoint:** `GET /analytics/websites/{id}` + +--- + +### Update Website + +**Endpoint:** `PUT /analytics/websites/{id}` + +--- + +### Delete Website + +**Endpoint:** `DELETE /analytics/websites/{id}` + +--- + +## Statistics + +### Get Website Stats + +**Endpoint:** `GET /analytics/websites/{id}/stats` + +**Query Parameters:** +- `start_date` - ISO 8601 date (default: 7 days ago) +- `end_date` - ISO 8601 date (default: now) +- `timezone` - Timezone string (default: UTC) + +**Response:** +```json +{ + "total_pageviews": 12543, + "unique_visitors": 3421, + "bounce_rate": 42.5, + "avg_session_duration": 185, + "period": { + "start": "2026-01-22T00:00:00Z", + "end": "2026-01-29T00:00:00Z" + } +} +``` + +--- + +### Get Time Series + +**Endpoint:** `GET /analytics/websites/{id}/timeseries` + +**Query Parameters:** +- `metric` - `pageviews`, `visitors`, `sessions` +- `start_date` - ISO 8601 date +- `end_date` - ISO 8601 date +- `interval` - `day`, `week`, `month` + +**Response:** +```json +{ + "2026-01-22": 1543, + "2026-01-23": 1621, + "2026-01-24": 1489 +} +``` + +--- + +### Get Real-time Stats + +**Endpoint:** `GET /analytics/websites/{id}/realtime` + +**Response:** +```json +{ + "active_visitors": 23, + "active_pages": [ + {"path": "/", "viewers": 12}, + {"path": "/pricing", "viewers": 8} + ], + "locations": [ + {"country_code": "GB", "visitors": 15}, + {"country_code": "US", "visitors": 8} + ], + "timestamp": "2026-01-29T12:00:00Z" +} +``` + +--- + +## Goals + +### List Goals + +**Endpoint:** `GET /analytics/websites/{id}/goals` + +--- + +### Create Goal + +**Endpoint:** `POST /analytics/websites/{id}/goals` + +**Request Body:** +```json +{ + "name": "Signup Completion", + "type": "pageview", + "match_type": "equals", + "match_value": "/signup/complete" +} +``` + +**Goal Types:** +- `pageview` - URL match +- `event` - Custom event match +- `duration` - Session duration threshold +- `pages_per_session` - Pageview count threshold + +**Match Types (for pageview):** +- `equals` - Exact match +- `contains` - Substring match +- `starts_with` - Prefix match +- `ends_with` - Suffix match +- `regex` - Regular expression + +--- + +### Get Goal Conversions + +**Endpoint:** `GET /analytics/websites/{id}/goals/{goalId}/conversions` + +**Query Parameters:** +- `start_date` +- `end_date` +- `limit` + +--- + +### Get Goal Stats + +**Endpoint:** `GET /analytics/websites/{id}/goals/{goalId}/conversions/stats` + +**Response:** +```json +{ + "total_conversions": 342, + "conversion_rate": 4.2, + "total_value": 15230.50, + "average_value": 44.53 +} +``` + +--- + +## Funnels + +### List Funnels + +**Endpoint:** `GET /analytics/websites/{id}/funnels` + +--- + +### Create Funnel + +**Endpoint:** `POST /analytics/websites/{id}/funnels` + +**Request Body:** +```json +{ + "name": "Checkout Funnel", + "is_strict": false, + "window_hours": 24 +} +``` + +--- + +### Get Funnel Analysis + +**Endpoint:** `GET /analytics/websites/{id}/funnels/{funnelId}/analysis` + +**Query Parameters:** +- `start_date` +- `end_date` + +**Response:** +```json +{ + "funnel_id": 1, + "funnel_name": "Checkout Funnel", + "summary": { + "total_entrants": 1000, + "completed": 120, + "completion_rate": 12.0, + "avg_completion_time": 540, + "total_steps": 4 + }, + "steps": [ + { + "step_id": 1, + "name": "Add to Cart", + "visitors": 1000, + "conversion_rate": 100.0, + "drop_off": 0, + "drop_off_rate": 0 + }, + { + "step_id": 2, + "name": "View Cart", + "visitors": 650, + "conversion_rate": 65.0, + "drop_off": 350, + "drop_off_rate": 35.0 + } + ] +} +``` + +--- + +### Add Funnel Step + +**Endpoint:** `POST /analytics/websites/{id}/funnels/{funnelId}/steps` + +**Request Body:** +```json +{ + "name": "Add to Cart", + "match_type": "pageview", + "match_value": "/cart/add", + "is_optional": false +} +``` + +--- + +## A/B Experiments + +### List Experiments + +**Endpoint:** `GET /analytics/websites/{id}/experiments` + +--- + +### Create Experiment + +**Endpoint:** `POST /analytics/websites/{id}/experiments` + +**Request Body:** +```json +{ + "name": "Button Colour Test", + "goal_type": "pageview", + "goal_value": "/signup/complete", + "traffic_percentage": 100 +} +``` + +--- + +### Start Experiment + +**Endpoint:** `POST /analytics/websites/{id}/experiments/{experimentId}/start` + +--- + +### Pause Experiment + +**Endpoint:** `POST /analytics/websites/{id}/experiments/{experimentId}/pause` + +--- + +### Stop Experiment + +**Endpoint:** `POST /analytics/websites/{id}/experiments/{experimentId}/stop` + +--- + +### Get Results + +**Endpoint:** `GET /analytics/websites/{id}/experiments/{experimentId}/results` + +**Response:** +```json +{ + "experiment": { + "id": 1, + "name": "Button Colour Test", + "status": "running" + }, + "metrics": { + "total_visitors": 5000, + "total_conversions": 250, + "overall_conversion_rate": 5.0 + }, + "analysis": { + "is_significant": true, + "confidence": 95.5, + "p_value": 0.045, + "winner": 2, + "recommendation": "\"Blue Button\" is the winner with 95% confidence (+12% lift)", + "results": { + "1": { + "name": "Control", + "visitors": 2500, + "conversions": 100, + "conversion_rate": 4.0, + "is_control": true + }, + "2": { + "name": "Blue Button", + "visitors": 2500, + "conversions": 150, + "conversion_rate": 6.0, + "lift": 50.0, + "is_significant": true + } + } + } +} +``` + +--- + +### Add Variant + +**Endpoint:** `POST /analytics/websites/{id}/experiments/{experimentId}/variants` + +**Request Body:** +```json +{ + "name": "Blue Button", + "is_control": false, + "weight": 50, + "config": { + "button_color": "#0066cc" + } +} +``` + +--- + +### Get Variant (Public) + +For client-side variant assignment. + +**Endpoint:** `GET /analytics/experiment/variant` + +**Query Parameters:** +- `experiment_id` +- `visitor_id` + +**Response:** +```json +{ + "variant_id": 2, + "variant_name": "Blue Button", + "config": { + "button_color": "#0066cc" + } +} +``` + +--- + +## Heatmaps + +### List Heatmaps + +**Endpoint:** `GET /analytics/websites/{id}/heatmaps` + +--- + +### Create Heatmap + +**Endpoint:** `POST /analytics/websites/{id}/heatmaps` + +**Request Body:** +```json +{ + "name": "Homepage Clicks", + "url_pattern": "/", + "type": "click" +} +``` + +**Heatmap Types:** +- `click` - Click positions +- `move` - Mouse movement +- `scroll` - Scroll depth + +--- + +### Get Heatmap Data + +**Endpoint:** `GET /analytics/websites/{id}/heatmaps/{heatmapId}/data` + +**Response:** +```json +{ + "heatmap_id": 1, + "data": [ + {"x": 500, "y": 300, "count": 45}, + {"x": 750, "y": 450, "count": 32} + ], + "viewport": { + "width": 1920, + "height": 1080 + } +} +``` + +--- + +## Session Replays + +### List Replays + +**Endpoint:** `GET /analytics/websites/{id}/replays` + +**Query Parameters:** +- `limit` +- `device_type` +- `country_code` + +--- + +### Get Replay + +**Endpoint:** `GET /analytics/websites/{id}/replays/{replayId}` + +--- + +### Get Playback Data + +**Endpoint:** `GET /analytics/websites/{id}/replays/{replayId}/playback` + +**Response:** rrweb-compatible event array + +--- + +### Delete Replay + +**Endpoint:** `DELETE /analytics/websites/{id}/replays/{replayId}` + +--- + +## Bot Detection + +### Get Bot Stats + +**Endpoint:** `GET /analytics/websites/{id}/bots/stats` + +**Response:** +```json +{ + "period": { + "from": "2026-01-01", + "to": "2026-01-29" + }, + "totals": { + "total_requests": 50000, + "blocked_requests": 2500, + "allowed_requests": 47500, + "block_rate": 5.0 + }, + "bot_types": { + "crawler": 1500, + "scraper": 800, + "headless": 200 + }, + "top_bots": { + "Googlebot": 800, + "Bingbot": 400, + "Ahrefs": 300 + } +} +``` + +--- + +### List Detections + +**Endpoint:** `GET /analytics/websites/{id}/bots/detections` + +--- + +### List Rules + +**Endpoint:** `GET /analytics/websites/{id}/bots/rules` + +--- + +### Create Rule + +**Endpoint:** `POST /analytics/websites/{id}/bots/rules` + +**Request Body:** +```json +{ + "rule_type": "whitelist", + "match_type": "ip", + "match_value": "192.168.1.100", + "description": "Office IP" +} +``` + +**Rule Types:** +- `whitelist` - Always allow +- `blacklist` - Always block + +**Match Types:** +- `ip` - Exact IP match +- `ip_range` - CIDR range (e.g., `192.168.1.0/24`) +- `user_agent` - Substring match in User-Agent + +--- + +## GDPR + +### Export Visitor Data + +**Endpoint:** `GET /analytics/gdpr/export/{visitorHash}` + +**Response:** JSON file download with all visitor data + +--- + +### Delete Visitor Data + +**Endpoint:** `DELETE /analytics/gdpr/visitor/{visitorHash}` + +**Response:** +```json +{ + "deleted_counts": { + "events": 152, + "sessions": 12, + "pageviews": 145, + "conversions": 3, + "visitor": 1 + } +} +``` + +--- + +### Anonymise Visitor + +**Endpoint:** `POST /analytics/gdpr/anonymise/{visitorHash}` + +Preserves aggregate data while removing PII. + +--- + +### Record Consent (Public) + +**Endpoint:** `POST /analytics/gdpr/consent` + +**Request Body:** +```json +{ + "visitor_id": "visitor-uuid", + "pixel_key": "uuid-pixel-key" +} +``` + +--- + +### Withdraw Consent (Public) + +**Endpoint:** `DELETE /analytics/gdpr/consent` + +--- + +### Check Consent Status (Public) + +**Endpoint:** `GET /analytics/gdpr/consent/status?visitor_id={id}&pixel_key={key}` + +--- + +## Email Reports + +### List Reports + +**Endpoint:** `GET /analytics/websites/{id}/email-reports` + +--- + +### Create Report + +**Endpoint:** `POST /analytics/websites/{id}/email-reports` + +**Request Body:** +```json +{ + "name": "Weekly Summary", + "frequency": "weekly", + "recipients": ["user@example.com"], + "metrics": ["pageviews", "visitors", "bounce_rate"] +} +``` + +**Frequencies:** +- `daily` +- `weekly` +- `monthly` + +--- + +### Preview Report + +**Endpoint:** `POST /analytics/websites/{id}/email-reports/{reportId}/preview` + +--- + +### Send Report Now + +**Endpoint:** `POST /analytics/websites/{id}/email-reports/{reportId}/send` + +--- + +## Error Responses + +### Standard Error Format + +```json +{ + "ok": false, + "error": "Error message", + "code": "ERROR_CODE" +} +``` + +### Common Error Codes + +| HTTP Status | Code | Description | +|-------------|------|-------------| +| 400 | `VALIDATION_ERROR` | Invalid request parameters | +| 401 | `UNAUTHENTICATED` | Missing or invalid authentication | +| 403 | `FORBIDDEN` | Insufficient permissions | +| 404 | `NOT_FOUND` | Resource not found | +| 429 | `RATE_LIMITED` | Too many requests | +| 500 | `SERVER_ERROR` | Internal server error | + +--- + +## SDK Integration + +### JavaScript Tracker + +```html + +``` + +### Server-Side Tracking + +```php +use Core\Mod\Analytics\Services\AnalyticsTrackingService; + +$tracking = app(AnalyticsTrackingService::class); +$tracking->track($website, [ + 'type' => 'pageview', + 'path' => '/api/endpoint', + 'visitor_id' => $request->header('X-Visitor-ID'), +], $request); +``` diff --git a/docs/packages/analytics/architecture.md b/docs/packages/analytics/architecture.md new file mode 100644 index 0000000..2779f74 --- /dev/null +++ b/docs/packages/analytics/architecture.md @@ -0,0 +1,419 @@ +--- +title: Architecture +description: Technical architecture of core-analytics +updated: 2026-01-29 +--- + +# Architecture + +This document describes the technical architecture of the core-analytics package, a privacy-focused website analytics module for the Core PHP Framework. + +## Overview + +core-analytics is a Laravel package providing website analytics with: + +- Privacy-first design (IP anonymisation, DNT respect, GDPR compliance) +- Real-time visitor tracking via Redis +- Session replays and heatmaps +- A/B testing with statistical significance +- Funnel analysis +- Bot detection and filtering +- Multi-tenant workspace isolation + +## Package Structure + +``` +core-analytics/ +├── Boot.php # Service provider, event registration +├── config.php # Configuration defaults +├── Controllers/ +│ ├── PixelController.php # Public tracking endpoints +│ └── Api/ # Authenticated API controllers +├── Services/ +│ ├── AnalyticsService.php # Core stats/aggregation +│ ├── AnalyticsTrackingService.php # Event tracking +│ ├── BotDetectionService.php # Bot scoring +│ ├── FunnelService.php # Funnel analysis +│ ├── GdprService.php # Privacy compliance +│ ├── GeoIpService.php # Geolocation +│ ├── RealtimeAnalyticsService.php # Redis-based realtime +│ ├── SessionReplayStorageService.php +│ └── AnalyticsExperimentService.php # A/B testing +├── Jobs/ +│ ├── ProcessTrackingEvent.php # Main event processor +│ ├── ProcessPageview.php +│ ├── ProcessHeatmapEvent.php +│ └── ProcessSessionReplay.php +├── Models/ # Eloquent models +├── Migrations/ # Database migrations +├── Console/Commands/ # Artisan commands +├── Mcp/Tools/ # MCP tool handlers +├── View/ # Blade/Livewire components +└── routes/ # Route definitions +``` + +## Event-Driven Module Loading + +The package follows the Core PHP Framework event-driven pattern: + +```php +class Boot extends ServiceProvider +{ + public static array $listens = [ + AdminPanelBooting::class => 'onAdminPanel', + WebRoutesRegistering::class => 'onWebRoutes', + ApiRoutesRegistering::class => 'onApiRoutes', + ConsoleBooting::class => 'onConsole', + ]; +} +``` + +Handlers are only instantiated when their events fire, enabling lazy loading. + +## Data Flow + +### Tracking Flow + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ JS Tracker │────>│ PixelController │────>│ ProcessTracking │ +│ (browser) │ │ (validation) │ │ Event (queue) │ +└─────────────────┘ └──────────────────┘ └────────┬────────┘ + │ + ┌─────────────────────────────────┼─────────────────────────────────┐ + │ │ │ + v v v + ┌───────────────┐ ┌──────────────────┐ ┌─────────────────┐ + │ Bot Check │ │ Entitlement │ │ GeoIP │ + │ (scoring) │ │ Check │ │ Lookup │ + └───────┬───────┘ └────────┬─────────┘ └────────┬────────┘ + │ │ │ + └───────────────────────────────┼──────────────────────────────────┘ + │ + v + ┌──────────────────┐ + │ Database Write │ + │ (visitor, │ + │ session, │ + │ event) │ + └────────┬─────────┘ + │ + ┌──────────────────────────────┼──────────────────────────────────┐ + │ │ │ + v v v + ┌───────────────┐ ┌──────────────────┐ ┌─────────────────┐ + │ Goal Check │ │ Cache Invalidate│ │ Realtime │ + │ (conversion) │ │ (stats cache) │ │ Broadcast │ + └───────────────┘ └──────────────────┘ └─────────────────┘ +``` + +### Statistics Query Flow + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Dashboard │────>│ Stats Controller│────>│ AnalyticsService│ +│ (request) │ │ (auth, scope) │ │ (query/cache) │ +└─────────────────┘ └──────────────────┘ └────────┬────────┘ + │ + v + ┌──────────────────┐ + │ Cache Check │ + │ (5 min TTL) │ + └────────┬─────────┘ + │ + ┌───────────────┴───────────────┐ + │ │ + v v + ┌───────────────┐ ┌───────────────┐ + │ Cache Hit │ │ Cache Miss │ + │ (return) │ │ (query DB) │ + └───────────────┘ └───────┬───────┘ + │ + v + ┌───────────────┐ + │ Store Cache │ + │ (return) │ + └───────────────┘ +``` + +## Database Schema + +### Core Tables + +| Table | Purpose | Volume | +|-------|---------|--------| +| `analytics_websites` | Website/property configuration | Low | +| `analytics_visitors` | Unique visitor records | High | +| `analytics_sessions` | Session aggregation | High | +| `analytics_events` | Individual events (pageviews, clicks) | Very High | +| `analytics_pageviews` | Denormalised pageview data | Very High | +| `analytics_goals` | Goal definitions | Low | +| `analytics_goal_conversions` | Goal conversion records | Medium | +| `analytics_daily_stats` | Pre-aggregated daily statistics | Low | + +### Feature Tables + +| Table | Purpose | +|-------|---------| +| `analytics_heatmaps` | Heatmap configuration | +| `analytics_heatmap_events` | Click/scroll/move coordinates | +| `analytics_session_replays` | Session replay metadata | +| `analytics_funnels` | Funnel definitions | +| `analytics_funnel_steps` | Funnel step configuration | +| `analytics_funnel_conversions` | Funnel progress tracking | +| `analytics_experiments` | A/B test configuration | +| `analytics_variants` | Experiment variant definitions | +| `analytics_experiment_visitors` | Variant assignments | +| `analytics_bot_detections` | Bot detection logs | +| `analytics_bot_rules` | Custom whitelist/blacklist | +| `analytics_bot_ip_cache` | IP reputation cache | +| `analytics_email_reports` | Scheduled report config | +| `analytics_email_report_logs` | Report send history | + +### Indexes + +Key indexes for query performance: + +```sql +-- Website + date range queries +INDEX (website_id, created_at) +INDEX (website_id, started_at) +INDEX (website_id, last_seen_at) + +-- Visitor/session lookups +UNIQUE (website_id, visitor_uuid) +UNIQUE (website_id, session_uuid) +INDEX (visitor_id) +INDEX (session_id) + +-- Path analysis +INDEX (website_id, path, created_at) +``` + +## Multi-Tenancy + +All models use the `BelongsToWorkspace` trait from core-tenant: + +```php +class AnalyticsWebsite extends Model +{ + use BelongsToWorkspace; + // ... +} +``` + +This provides: +- Automatic `workspace_id` assignment on create +- Global scope filtering to current workspace +- `MissingWorkspaceContextException` safety + +## Caching Strategy + +### Cache Keys + +``` +analytics:stats:{website_id}:{start}:{end} +analytics:timeseries:{website_id}:{metric}:{start}:{end}:{interval} +analytics:top_pages:{website_id}:{start}:{end}:{limit} +analytics:traffic_sources:{website_id}:{start}:{end} +analytics:geo:{website_id}:{start}:{end} +analytics_website:{pixel_key} # Website lookup cache (1 hour) +analytics_config:{pixel_key} # Config cache (5 minutes) +bot_rules:{website_id} # Bot rules cache (5 minutes) +``` + +### Invalidation + +Cache invalidation occurs on: +- New tracking event (debounced, selective invalidation) +- Manual invalidation via `AnalyticsService::invalidateCache()` +- TTL expiration (5 minutes for stats) + +## Queue Architecture + +### Queues + +| Queue | Purpose | Workers | +|-------|---------|---------| +| `analytics-tracking` | High-priority tracking events | 2-4 | +| `analytics` | Heatmaps, replays, cleanup | 1-2 | +| `default` | Email reports, low-priority | 1 | + +### Job Configuration + +```php +class ProcessTrackingEvent implements ShouldQueue +{ + public int $tries = 3; + public int $timeout = 30; +} +``` + +## Real-Time Analytics + +Uses Redis sorted sets for 5-minute sliding window: + +``` +analytics:realtime:visitors:{website_id} # Sorted set: visitor_id -> timestamp +analytics:realtime:visitor_page:{website_id}:{id} # String: current path +analytics:realtime:visitor_country:{website_id}:{id} +``` + +### Broadcast Throttling + +Updates are throttled to 2-second intervals to prevent flooding WebSocket channels during high traffic. + +## Bot Detection + +### Scoring Algorithm + +Signals are weighted (sum = 100): + +| Signal | Weight | Description | +|--------|--------|-------------| +| User-Agent | 35% | Bot patterns, headless browsers, HTTP libraries | +| Headers | 20% | Missing Accept/Accept-Language, automation indicators | +| IP Reputation | 15% | Datacenter IPs, known crawler ranges | +| Behaviour | 20% | JS indicators, screen dimensions, timing | +| Custom Rules | 10% | Whitelist/blacklist matches | + +### Thresholds + +- `threshold` (50): Score >= threshold = classified as bot +- `block_threshold` (70): Score >= threshold = blocked from tracking +- `min_log_score` (30): Minimum score to log detection + +### Legitimate Crawlers + +Known search engine IPs (Google, Bing, etc.) receive a 30-point score reduction and are logged but not blocked. + +## A/B Testing + +### Variant Assignment + +Uses deterministic hashing for consistent assignment: + +```php +$hash = abs(crc32($visitorId . $experimentId)) % 100; +``` + +This ensures the same visitor always gets the same variant, even across sessions. + +### Statistical Significance + +Uses two-proportion z-test: + +1. Calculate conversion rates (p1, p2) +2. Calculate pooled proportion +3. Calculate standard error +4. Compute z-score and p-value +5. Compare against confidence level (default 95%) + +Minimum sample size per variant: 100 (configurable). + +## Data Retention + +Tier-based retention: + +| Tier | Days | +|------|------| +| Free | 30 | +| Pro | 90 | +| Business | 365 | +| Enterprise | 3650 | + +Cleanup via `analytics:cleanup` command: +1. Aggregate data into `analytics_daily_stats` +2. Delete raw events/sessions/pageviews +3. Clean orphaned visitors + +## Privacy Features + +### IP Anonymisation + +Last octet zeroed by default: +``` +192.168.1.123 -> 192.168.1.0 +``` + +### Do Not Track + +Respects DNT header when `analytics.privacy.respect_dnt` is enabled. + +### GDPR Compliance + +- `GdprService::exportVisitorData()` - Full data export +- `GdprService::deleteVisitorData()` - Complete deletion +- `GdprService::anonymiseVisitor()` - Preserve aggregates, remove PII +- Consent tracking per-visitor + +## External Dependencies + +### Required + +- `host-uk/core` - Core PHP Framework +- Redis - Real-time analytics, caching +- Queue worker - Event processing + +### Optional + +- MaxMind GeoLite2 - IP geolocation (falls back to CDN headers) +- S3/compatible - Session replay storage (falls back to local) + +## Configuration + +Key configuration options in `config.php`: + +```php +return [ + 'session_replay' => [ + 'disk' => 'local', // or 's3' + 'expiry_days' => 90, + 'max_size' => 10 * 1024 * 1024, + ], + 'bot_detection' => [ + 'enabled' => true, + 'threshold' => 50, + 'block_threshold' => 70, + ], + 'privacy' => [ + 'anonymise_ip' => true, + 'respect_dnt' => true, + ], + 'retention' => [ + 'tiers' => [ + 'free' => 30, + 'pro' => 90, + 'business' => 365, + 'enterprise' => 3650, + ], + ], +]; +``` + +## Scaling Considerations + +### High Volume Sites + +For sites with >1M daily pageviews: + +1. **Separate analytics database** - Isolate from application DB +2. **Read replicas** - Route stats queries to replicas +3. **Redis Cluster** - Scale real-time tracking +4. **Dedicated queue workers** - Scale event processing +5. **ClickHouse** - Consider columnar storage for aggregations + +### Recommended Worker Configuration + +``` +# Low traffic (<100k/day) +php artisan queue:work --queue=analytics-tracking,analytics + +# Medium traffic (100k-1M/day) +php artisan queue:work --queue=analytics-tracking --workers=2 +php artisan queue:work --queue=analytics + +# High traffic (>1M/day) +php artisan queue:work --queue=analytics-tracking --workers=4 +php artisan queue:work --queue=analytics --workers=2 +``` diff --git a/docs/packages/analytics/security.md b/docs/packages/analytics/security.md new file mode 100644 index 0000000..d7f59b0 --- /dev/null +++ b/docs/packages/analytics/security.md @@ -0,0 +1,461 @@ +--- +title: Security +description: Security considerations and audit notes for core-analytics +updated: 2026-01-29 +--- + +# Security + +This document outlines security considerations, implemented mitigations, and audit notes for the core-analytics package. + +## Threat Model + +### Actors + +1. **Anonymous attackers** - Attempting to exploit public tracking endpoints +2. **Authenticated users** - Potentially abusing legitimate access +3. **Tracked visitors** - Privacy concerns, data exposure +4. **Malicious websites** - Cross-site attacks via tracking pixel + +### Assets + +1. Analytics data (pageviews, sessions, conversions) +2. Visitor PII (IP addresses, geolocation, device info) +3. Session replay recordings +4. Workspace/tenant data isolation + +## Implemented Mitigations + +### Input Validation + +#### Tracking Endpoints + +All tracking endpoints validate input: + +```php +$validated = $request->validate([ + 'key' => 'required|uuid', + 'type' => 'sometimes|string|in:pageview,click,scroll,form,goal,custom', + 'path' => 'required|string|max:512', + 'title' => 'sometimes|string|max:256', + // ... more fields +]); +``` + +**Audit note:** Consider adding Form Request classes for better organisation. + +#### Regex Pattern Safety + +Goal patterns support regex matching with ReDoS protection: + +```php +protected function safeRegexMatch(string $pattern, string $subject): bool +{ + // Limit subject length + if (strlen($subject) > 2048) { + $subject = substr($subject, 0, 2048); + } + + // Validate pattern syntax + if (!$this->isValidRegexPattern($pattern)) { + return false; + } + + // Set lower backtrack limit + ini_set('pcre.backtrack_limit', '10000'); + // ... +} +``` + +**Audit note:** Consider pattern complexity analysis at storage time, not just execution. + +### Rate Limiting + +Public tracking endpoints are rate-limited: + +```php +Route::middleware('throttle:10000,1')->prefix('analytics')->group(function () { + // 10,000 requests per minute +}); +``` + +**Audit note:** Rate limiting is per-IP, not per-pixel-key. Shared pool could be exhausted by targeting one key. Consider per-key quotas. + +### Bot Detection + +Multi-signal bot detection prevents analytics pollution: + +| Signal | Detection Method | +|--------|------------------| +| User-Agent | Pattern matching for known bots, headless browsers | +| HTTP Headers | Missing Accept/Accept-Language headers | +| IP Reputation | Datacenter IP ranges, cached bot scores | +| Behaviour | Invalid screen dimensions, fast request timing | + +**Audit note:** Bot detection relies on IP caching. NAT/VPN users could be incorrectly flagged. Consider composite scoring. + +### Multi-Tenant Isolation + +All models use `BelongsToWorkspace` trait: + +```php +class AnalyticsWebsite extends Model +{ + use BelongsToWorkspace; + // Automatically scoped to current workspace +} +``` + +This provides: +- Automatic `workspace_id` assignment on create +- Global scope filtering queries to current workspace +- Exception thrown if no workspace context + +**Audit note:** Verify all API endpoints properly set workspace context before queries. + +### IP Anonymisation + +IP addresses are anonymised before storage by default: + +```php +'ip' => PrivacyHelper::anonymiseIp($request->ip()), +``` + +Zeroes the last octet: `192.168.1.123` -> `192.168.1.0` + +Controlled by `analytics.privacy.anonymise_ip` config option. + +### Pixel Key Security + +Pixel keys are UUIDs generated at website creation: + +```php +protected static function booted(): void +{ + static::creating(function (AnalyticsWebsite $website) { + if (empty($website->pixel_key)) { + $website->pixel_key = Str::uuid(); + } + }); +} +``` + +Keys are: +- Unique across all websites +- Cannot be user-specified +- Cached for lookup efficiency + +**Audit note:** Keys are not rotatable. Consider adding key rotation capability. + +### CORS Configuration + +Tracking endpoints allow cross-origin requests (required for tracking pixels): + +```php +protected function corsHeaders(): array +{ + return [ + 'Access-Control-Allow-Origin' => '*', + 'Access-Control-Allow-Methods' => 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers' => 'Content-Type, X-Requested-With', + 'Access-Control-Max-Age' => '86400', + ]; +} +``` + +**Audit note:** `*` origin is acceptable for tracking endpoints. Authenticated API endpoints should have stricter CORS. + +## GDPR Compliance + +### Data Export + +Full data export for GDPR requests: + +```php +public function exportVisitorData(AnalyticsVisitor $visitor): array +{ + return [ + 'visitor' => [...], + 'sessions' => $this->exportSessions($visitor), + 'events' => $this->exportEvents($visitor), + 'pageviews' => $this->exportPageviews($visitor), + 'conversions' => $this->exportConversions($visitor), + 'heatmap_events' => $this->exportHeatmapEvents($visitor), + 'session_replays' => $this->exportSessionReplays($visitor), + ]; +} +``` + +**Audit note:** Verify `AnalyticsFunnelConversion` and `AnalyticsExperimentVisitor` are included. + +### Data Deletion + +Complete data deletion: + +```php +public function deleteVisitorData(AnalyticsVisitor $visitor): array +{ + DB::transaction(function () use ($visitor, &$counts) { + $counts['events'] = AnalyticsEvent::where('visitor_id', $visitor->id)->delete(); + $counts['pageviews'] = Pageview::where('visitor_id', $visitor->visitor_uuid)->delete(); + // ... all related data + $counts['visitor'] = $visitor->delete() ? 1 : 0; + }); +} +``` + +### Anonymisation + +Alternative to deletion that preserves aggregates: + +```php +public function anonymiseVisitor(AnalyticsVisitor $visitor): AnalyticsVisitor +{ + $visitor->update([ + 'ip' => null, + 'country_code' => null, + 'city_name' => null, + 'region' => null, + 'browser_language' => null, + 'custom_parameters' => null, + 'is_anonymised' => true, + 'anonymised_at' => now(), + ]); + // Also anonymises related records +} +``` + +### Consent Tracking + +Per-visitor consent tracking: + +```php +public function recordConsent(AnalyticsVisitor $visitor, ?string $ip = null): AnalyticsVisitor +{ + $visitor->update([ + 'consent_given' => true, + 'consent_given_at' => now(), + 'consent_ip' => $ip, + ]); +} +``` + +Tracking can be configured to require consent: + +```php +if ($website->require_consent && !$visitor->hasConsent()) { + return null; // Don't track +} +``` + +## Session Replay Security + +### Storage + +Replays are stored compressed on configured disk (S3 recommended for production): + +```php +$compressedData = gzencode($jsonData, 9); +Storage::disk($this->disk)->put($storagePath, $compressedData); +``` + +### Sensitive Data + +**Warning:** Session replays may capture sensitive form data. + +Mitigations: +1. Client-side SDK should mask password fields, credit cards +2. Max size limit (10MB default) prevents large data exfiltration +3. Expiry (90 days default) limits data retention + +**Audit note:** Consider server-side PII detection and redaction. + +### Access Control + +Replay playback requires authentication and workspace ownership: + +```php +Route::middleware(['auth', 'api'])->group(function () { + Route::get('/websites/{website}/replays/{replay}/playback', ...); +}); +``` + +## A/B Testing Security + +### Variant Assignment + +Deterministic assignment prevents manipulation: + +```php +$hash = abs(crc32($visitorId . $experimentId)) % 100; +``` + +Users cannot choose their variant. + +### Statistical Integrity + +- Minimum sample sizes enforced before declaring winners +- Statistical significance uses standard z-test +- Results are read-only once experiment is complete + +## API Security + +### Authentication + +All management endpoints require authentication: + +```php +Route::middleware(['auth', 'api'])->prefix('analytics')->group(function () { + // Websites, goals, experiments, etc. +}); +``` + +### Authorisation + +Workspace scoping provides implicit authorisation. Users can only access resources in their workspace. + +**Audit note:** Consider explicit policy classes for finer-grained control. + +## Data Retention + +### Automatic Cleanup + +```bash +php artisan analytics:cleanup +``` + +Tier-based retention: + +| Tier | Days | +|------|------| +| Free | 30 | +| Pro | 90 | +| Business | 365 | +| Enterprise | 3650 | + +### Aggregation Before Deletion + +Data is aggregated into `analytics_daily_stats` before raw data deletion, preserving historical trends. + +## Known Vulnerabilities / Limitations + +### Rate Limit Bypass + +**Issue:** Rate limiting uses shared pool across all pixel keys. + +**Impact:** Medium - Attacker could exhaust quota for legitimate users. + +**Mitigation:** Consider per-pixel-key rate limiting. + +### Bot IP Cache Poisoning + +**Issue:** IP scores are cached for 24 hours. NAT/VPN users share IPs. + +**Impact:** Low - False positives for legitimate users behind bot-flagged IPs. + +**Mitigation:** Consider composite scoring (IP + fingerprint) and score decay. + +### Session Replay PII + +**Issue:** Replays may contain sensitive data if client SDK doesn't mask inputs. + +**Impact:** Medium - PII exposure in replays. + +**Mitigation:** Document client-side requirements; consider server-side sanitisation. + +## Security Headers + +The package doesn't set security headers (handled by framework/proxy). Recommended: + +``` +X-Content-Type-Options: nosniff +X-Frame-Options: DENY +Content-Security-Policy: default-src 'self' +``` + +## Logging + +Security-relevant events are logged: + +```php +Log::info('GDPR: Visitor data deleted', [...]); +Log::info('GDPR: Consent recorded', [...]); +Log::warning('ProcessTrackingEvent: Missing website_id', [...]); +``` + +**Audit note:** Consider structured security event logging for SIEM integration. + +## Dependency Security + +### Direct Dependencies + +- `host-uk/core` - Internal, audited +- `laravel/*` - Well-maintained, security updates + +### GeoIP Database + +MaxMind GeoLite2 database should be updated regularly: + +```bash +# Recommended: weekly cron job +wget -O storage/app/geoip/GeoLite2-City.mmdb https://... +``` + +## Recommendations + +### High Priority + +1. Implement per-pixel-key rate limiting +2. Add Form Request classes for input validation +3. Server-side session replay PII detection +4. Complete GDPR export (funnel/experiment data) + +### Medium Priority + +1. Add policy classes for explicit authorisation +2. Pixel key rotation capability +3. Composite bot scoring (IP + fingerprint) +4. Structured security event logging + +### Low Priority + +1. IP reputation decay +2. Anomaly detection for abuse patterns +3. CSP header configuration guide + +## Audit Checklist + +- [ ] All models use BelongsToWorkspace trait +- [ ] API endpoints set workspace context +- [ ] Input validation on all endpoints +- [ ] Rate limiting active +- [ ] GDPR export complete +- [ ] Session replay PII handling documented +- [ ] Bot detection thresholds tuned +- [ ] Cleanup job scheduled +- [ ] GeoIP database current +- [ ] Dependencies updated + +## Incident Response + +### Data Breach + +1. Identify affected workspaces +2. Export affected visitor data +3. Notify workspace owners +4. Delete or anonymise exposed data +5. Rotate affected pixel keys + +### Bot Attack + +1. Review `analytics_bot_detections` for patterns +2. Add custom blacklist rules +3. Adjust thresholds if needed +4. Consider IP-based blocking at CDN level + +### Quota Abuse + +1. Identify abusive pixel key +2. Check workspace entitlements +3. Disable tracking for affected website +4. Contact workspace owner diff --git a/docs/packages/analytics/testing.md b/docs/packages/analytics/testing.md new file mode 100644 index 0000000..bc37900 --- /dev/null +++ b/docs/packages/analytics/testing.md @@ -0,0 +1,435 @@ +--- +title: Testing +description: Test coverage and testing guide for core-analytics +updated: 2026-01-29 +--- + +# Testing + +This document describes the testing strategy, coverage, and guidelines for the core-analytics package. + +## Test Structure + +``` +tests/ +├── Feature/ +│ ├── Admin/ # Admin panel tests +│ ├── Api/ # API endpoint tests +│ ├── Integration/ # End-to-end flow tests +│ ├── Mcp/ # MCP tool tests +│ ├── AnalyticsServiceTest.php +│ ├── AnalyticsTrackingServiceTest.php +│ ├── BotDetectionServiceTest.php +│ ├── ExperimentTest.php +│ ├── FunnelTest.php +│ ├── GdprTest.php +│ ├── GoalApiTest.php +│ ├── GoalTest.php +│ ├── HeatmapTest.php +│ ├── SessionReplayTest.php +│ └── ... +├── Unit/ +│ └── UserAgentParserTest.php +├── UseCase/ +│ ├── CreateWebsiteBasic.php +│ └── CreateWebsiteEnhanced.php +└── TestCase.php +``` + +## Running Tests + +```bash +# Run all tests +composer test + +# Run with coverage +composer test -- --coverage + +# Run specific test file +./vendor/bin/pest tests/Feature/BotDetectionServiceTest.php + +# Run specific test +./vendor/bin/pest --filter="test_detects_known_bot_user_agents" + +# Run tests in parallel +./vendor/bin/pest --parallel +``` + +## Test Database + +Tests use SQLite in-memory database with `RefreshDatabase` trait: + +```php +use Illuminate\Foundation\Testing\RefreshDatabase; + +class BotDetectionServiceTest extends TestCase +{ + use RefreshDatabase; +} +``` + +## Coverage Summary + +### Services + +| Service | Coverage | Notes | +|---------|----------|-------| +| AnalyticsService | Good | Stats generation, time series | +| AnalyticsTrackingService | Good | Event tracking, session management | +| BotDetectionService | Good | Detection, caching, rules | +| FunnelService | Partial | Analysis covered, step matching needs more | +| GdprService | Good | Export, delete, anonymise | +| GeoIpService | Partial | CDN headers covered, MaxMind mocked | +| RealtimeAnalyticsService | Partial | Redis operations, needs integration test | +| SessionReplayStorageService | Partial | Store/retrieve, cleanup needs more | +| AnalyticsExperimentService | Good | Variant assignment, significance | + +### Controllers + +| Controller | Coverage | Notes | +|------------|----------|-------| +| PixelController | Partial | Basic tracking, needs edge cases | +| AnalyticsWebsiteController | Basic | CRUD operations | +| AnalyticsStatsController | Basic | Endpoint tests | +| GoalController | Good | CRUD and conversions | +| ExperimentController | Good | Full lifecycle | +| FunnelController | Partial | Analysis endpoint | +| GdprController | Partial | Export tested | +| BotDetectionController | Basic | Stats endpoint | + +### Models + +| Model | Coverage | Notes | +|-------|----------|-------| +| AnalyticsWebsite | Good | Relationships, scopes | +| AnalyticsVisitor | Partial | Basic tests | +| AnalyticsSession | Partial | Basic tests | +| AnalyticsEvent | Partial | Basic tests | +| Goal | Good | Matching logic, conversions | +| AnalyticsFunnel | Partial | Basic tests | +| AnalyticsExperiment | Good | Lifecycle, variants | +| BotDetection | Good | Logging tests | +| BotRule | Good | Matching tests | + +## Testing Patterns + +### Service Tests + +```php +describe('Analytics Service', function () { + beforeEach(function () { + Cache::flush(); + $this->service = new AnalyticsService; + $this->website = Website::factory()->create(); + }); + + it('generates basic statistics for a website', function () { + AnalyticsEvent::factory()->count(10)->create([ + 'website_id' => $this->website->id, + 'type' => 'pageview', + ]); + + $stats = $this->service->generateStats($this->website); + + expect($stats)->toHaveKeys([ + 'total_pageviews', + 'unique_visitors', + 'bounce_rate', + ]); + }); +}); +``` + +### Bot Detection Tests + +```php +public function test_detects_known_bot_user_agents(): void +{ + $botUserAgents = [ + 'Googlebot/2.1 (+http://www.google.com/bot.html)', + 'curl/7.79.1', + 'HeadlessChrome/120.0.6099.109', + ]; + + foreach ($botUserAgents as $ua) { + $request = $this->createRequest($ua); + $result = $this->service->analyse($request); + + $this->assertTrue( + $result['is_bot'] || $result['score'] >= 30, + "Failed to detect bot: {$ua}" + ); + } +} +``` + +### API Tests + +```php +it('returns website statistics', function () { + $website = AnalyticsWebsite::factory()->create(); + + $this->actingAs($user) + ->getJson("/analytics/websites/{$website->id}/stats") + ->assertOk() + ->assertJsonStructure([ + 'total_pageviews', + 'unique_visitors', + 'bounce_rate', + ]); +}); +``` + +### Mock Request Helper + +```php +protected function createRequest(string $userAgent, array $headers = []): Request +{ + $request = Request::create('/', 'GET'); + $request->headers->set('User-Agent', $userAgent); + + foreach ($headers as $name => $value) { + $request->headers->set($name, $value); + } + + $request->server->set('REMOTE_ADDR', '127.0.0.1'); + + return $request; +} +``` + +## Missing Test Coverage + +### High Priority + +1. **Full tracking flow integration test** + - Pixel hit -> Queue job -> Database write -> Cache invalidation + - Goal conversion flow + - Experiment variant assignment + +2. **Rate limiting tests** + - Verify rate limits are enforced + - Test different throttle scenarios + +3. **Multi-tenant isolation tests** + - Verify workspace scoping + - Cross-workspace data leak prevention + +### Medium Priority + +1. **Bot detection edge cases** + - Privacy browsers (Brave, Tor) + - Corporate proxies + - VPN users + +2. **Session replay tests** + - Large replay handling + - Compression/decompression + - Expiry cleanup + +3. **Real-time analytics tests** + - Redis sorted set operations + - Broadcast throttling + - Cleanup of stale data + +### Low Priority + +1. **Email report tests** + - Report generation + - Scheduling + - Preview functionality + +2. **Heatmap aggregation tests** + - Large dataset aggregation + - Viewport normalisation + +## Test Data Factories + +### AnalyticsWebsiteFactory + +```php +AnalyticsWebsite::factory()->create([ + 'name' => 'Test Site', + 'host' => 'test.example.com', + 'tracking_enabled' => true, +]); +``` + +### AnalyticsEventFactory + +```php +AnalyticsEvent::factory()->count(100)->create([ + 'website_id' => $website->id, + 'type' => 'pageview', + 'created_at' => now()->subDays(rand(1, 30)), +]); +``` + +### AnalyticsSessionFactory + +```php +AnalyticsSession::factory()->create([ + 'website_id' => $website->id, + 'is_bounce' => false, + 'duration' => 300, + 'pageviews' => 5, +]); +``` + +## Mocking External Services + +### Redis + +```php +use Illuminate\Support\Facades\Redis; + +Redis::shouldReceive('zadd')->once(); +Redis::shouldReceive('zrangebyscore')->andReturn(['visitor-1', 'visitor-2']); +``` + +### MaxMind GeoIP + +```php +$this->mock(GeoIpService::class, function ($mock) { + $mock->shouldReceive('lookup') + ->andReturn([ + 'country_code' => 'GB', + 'city_name' => 'London', + ]); +}); +``` + +### Queue Jobs + +```php +use Illuminate\Support\Facades\Queue; + +Queue::fake(); + +// Perform action that dispatches job + +Queue::assertPushed(ProcessTrackingEvent::class); +``` + +## Performance Testing + +For high-volume testing: + +```php +it('handles high volume of events efficiently', function () { + $website = AnalyticsWebsite::factory()->create(); + + // Create 10,000 events + AnalyticsEvent::factory() + ->count(10000) + ->create(['website_id' => $website->id]); + + $start = microtime(true); + $stats = $this->service->generateStats($website); + $duration = microtime(true) - $start; + + expect($duration)->toBeLessThan(1.0); // Under 1 second +}); +``` + +## Test Environment Configuration + +### phpunit.xml + +```xml + + + + + + + + +``` + +### Test-Specific Config + +```php +config(['analytics.bot_detection.enabled' => true]); +config(['analytics.bot_detection.threshold' => 50]); +``` + +## CI Integration + +Tests run on GitHub Actions: + +```yaml +- name: Run Tests + run: composer test -- --coverage-clover coverage.xml + +- name: Upload Coverage + uses: codecov/codecov-action@v3 +``` + +## Writing New Tests + +### Guidelines + +1. Use Pest syntax for new tests +2. Use descriptive test names +3. Test one thing per test +4. Use factories for test data +5. Clean up after tests (RefreshDatabase handles this) +6. Mock external services +7. Test edge cases and error conditions + +### Example Test Structure + +```php +describe('Feature Name', function () { + beforeEach(function () { + // Setup + }); + + describe('scenario A', function () { + it('does expected behaviour', function () { + // Arrange + $input = [...]; + + // Act + $result = $this->service->method($input); + + // Assert + expect($result)->toBe($expected); + }); + + it('handles edge case', function () { + // ... + }); + + it('throws exception for invalid input', function () { + expect(fn() => $this->service->method(null)) + ->toThrow(InvalidArgumentException::class); + }); + }); +}); +``` + +## Debugging Tests + +### Dump and Die + +```php +dd($result); // Dump and die +dump($result); // Dump and continue +``` + +### Database Queries + +```php +\DB::enableQueryLog(); +// ... code ... +dd(\DB::getQueryLog()); +``` + +### Test Output + +```bash +./vendor/bin/pest --verbose +./vendor/bin/pest --debug +``` diff --git a/docs/packages/bio/architecture.md b/docs/packages/bio/architecture.md new file mode 100644 index 0000000..f453093 --- /dev/null +++ b/docs/packages/bio/architecture.md @@ -0,0 +1,396 @@ +--- +title: Architecture +description: Technical architecture of the core-bio package +updated: 2026-01-29 +--- + +# core-bio Architecture + +This document describes the technical architecture of the `core-bio` package, which provides link-in-bio, short link, and static page functionality for the Host UK platform. + +## Overview + +The `core-bio` package is a Laravel package that integrates with the Core PHP Framework's event-driven module system. It provides: + +- **Biolink Pages** - Block-based link-in-bio pages with 60+ block types +- **Short Links** - URL shortening with redirect tracking +- **Static Pages** - Custom HTML/CSS/JS pages with XSS sanitisation +- **vCards** - Downloadable contact cards +- **Event Pages** - Calendar event links with iCal generation +- **File Links** - Secure file downloads with tracking + +## Module Registration + +The package uses event-driven registration via `Boot.php`: + +```php +public static array $listens = [ + ClientRoutesRegistering::class => 'onClientRoutes', + WebRoutesRegistering::class => 'onWebRoutes', + ApiRoutesRegistering::class => 'onApiRoutes', +]; +``` + +This lazy-loading pattern means the module is only instantiated when its events fire. + +## Directory Structure + +``` +core-bio/ +├── Actions/ # Single-purpose business logic (CreateBiolink, UpdateBiolink, DeleteBiolink) +├── Console/ +│ └── Commands/ # Artisan commands (aggregation, cleanup, domain verification) +├── Controllers/ +│ ├── Api/ # REST API controllers (PageController, BlockController, etc.) +│ └── Web/ # Web controllers (public rendering, redirects, submissions) +├── Effects/ +│ ├── Background/ # Background effect implementations (snow, leaves, stars, etc.) +│ ├── Block/ # Block-level effects +│ └── Catalog.php # Effect registry +├── Exceptions/ # Custom exceptions (AI service errors) +├── Jobs/ # Queue jobs (click tracking, notifications) +├── Lang/ # Translation files (en_GB) +├── Mail/ # Mailable classes (BioReport) +├── Mcp/ +│ └── Tools/ # MCP AI agent tools +├── Middleware/ # HTTP middleware (domain resolution, targeting, password) +├── Migrations/ # Database migrations +├── Models/ # Eloquent models +├── Notifications/ # Laravel notifications +├── Policies/ # Authorisation policies +├── Requests/ # Form request validation +├── Services/ # Business logic services +├── View/ +│ ├── Blade/ # Blade templates +│ │ ├── admin/ # Admin panel views +│ │ ├── components/ # Reusable components +│ │ └── emails/ # Email templates +│ └── Modal/ +│ └── Admin/ # Livewire admin components +├── routes/ # Route definitions (web.php, api.php, console.php) +├── Boot.php # Service provider and event handlers +├── config.php # Package configuration (merged as 'webpage') +└── device-frames.php # Device frame configuration for previews +``` + +## Core Models + +### Page (Biolink) + +The central model representing all page types. Uses single-table inheritance via `type` column. + +```php +// Types: 'biolink', 'link' (short link), 'static', 'vcard', 'event', 'file' +$biolink = Page::create([ + 'workspace_id' => $workspace->id, + 'user_id' => $user->id, + 'type' => 'biolink', + 'url' => 'mypage', + 'settings' => [...], +]); +``` + +**Key relationships:** +- `workspace()` - Multi-tenant isolation via `BelongsToWorkspace` trait +- `blocks()` - HasMany Block for biolink pages +- `theme()` - BelongsTo Theme for styling +- `domain()` - BelongsTo Domain for custom domains +- `project()` - BelongsTo Project for organisation +- `pixels()` - BelongsToMany Pixel for tracking +- `revisions()` - HasMany BioRevision for undo functionality +- `subPages()` - HasMany self-referential for nested pages + +### Block + +Represents a content block within a biolink page. Supports 60+ block types across categories: + +- **Standard**: link, heading, paragraph, avatar, image, socials +- **Embeds**: youtube, spotify, tiktok, vimeo, etc. +- **Advanced**: map, email_collector, faq, countdown, etc. +- **Payments**: paypal, donation, product, service + +```php +$block = Block::create([ + 'biolink_id' => $biolink->id, + 'type' => 'link', + 'region' => 'content', // HLCRF region + 'order' => 1, + 'settings' => [ + 'url' => 'https://example.com', + 'text' => 'Visit Example', + ], +]); +``` + +**A/B Testing Fields:** +- `ab_test_id` - UUID grouping variants +- `is_control` - Whether this is the control variant +- `traffic_split` - Percentage of traffic for this variant +- `is_winner` - Declared winner after test + +### Click / ClickStat + +Two-tier analytics storage: + +- **Click** - Individual click records with full attribution (IP hash, country, device, referrer, UTM) +- **ClickStat** - Pre-aggregated daily statistics for fast queries + +Clicks are tracked asynchronously via `TrackBioLinkClick` job to avoid blocking page loads. + +## HLCRF Layout System + +The package supports multi-region layouts (Header, Left, Content, Right, Footer) with per-breakpoint configuration: + +```php +// Layout presets from config +'layout_presets' => [ + 'bio' => ['phone' => 'C', 'tablet' => 'C', 'desktop' => 'C'], + 'landing' => ['phone' => 'C', 'tablet' => 'HCF', 'desktop' => 'HCF'], + 'blog' => ['phone' => 'C', 'tablet' => 'HCF', 'desktop' => 'HCRF'], + 'docs' => ['phone' => 'C', 'tablet' => 'HCF', 'desktop' => 'HLCF'], + 'portfolio' => ['phone' => 'C', 'tablet' => 'HCF', 'desktop' => 'HLCRF'], +], +``` + +Blocks specify their region and per-region ordering: + +```php +$block->region = Block::REGION_HEADER; // 'header', 'left', 'content', 'right', 'footer' +$block->region_order = 1; +``` + +## Service Layer + +### AnalyticsService + +Queries click data with retention enforcement: + +```php +$service = app(AnalyticsService::class); + +// Respects workspace entitlements for data retention +$retention = $service->enforceDateRetention($start, $end, $workspace); + +// Get breakdown data +$byCountry = $service->getClicksByCountry($biolink, $start, $end); +$byDevice = $service->getClicksByDevice($biolink, $start, $end); +$byReferrer = $service->getClicksByReferrer($biolink, $start, $end); +``` + +### StaticPageSanitiser + +Security-critical service for sanitising user-provided HTML/CSS/JS: + +```php +$sanitiser = app(StaticPageSanitiser::class); + +$clean = $sanitiser->sanitiseStaticPage( + html: $userHtml, + css: $userCss, + js: $userJs +); +``` + +**Security approach:** +- HTML: HTMLPurifier with strict allowlist +- CSS: Blocklist for dangerous patterns (expression, javascript:, @import) +- JS: Blocklist for eval-like constructs (documented limitations) + +See `docs/security.md` for details. + +### DomainVerificationService + +Handles custom domain DNS verification: + +```php +$service = app(DomainVerificationService::class); + +// Verify via TXT record or CNAME +$verified = $service->verify($domain); + +// Get DNS instructions for user +$instructions = $service->getDnsInstructions($domain); +``` + +### BioPasswordRateLimiter + +Prevents brute force attacks on password-protected pages: + +```php +$limiter = app(BioPasswordRateLimiter::class); + +if ($limiter->tooManyAttempts($biolink, $request)) { + $seconds = $limiter->availableIn($biolink, $request); + // Show rate limit error +} + +// On failed attempt - increments with exponential backoff +$limiter->increment($biolink, $request); + +// On success - clear rate limit (backoff level persists) +$limiter->clear($biolink, $request); +``` + +## API Layer + +RESTful API supporting both session auth and API key auth: + +``` +GET /api/bio # List biolinks +POST /api/bio # Create biolink +GET /api/bio/{id} # Get biolink +PUT /api/bio/{id} # Update biolink +DELETE /api/bio/{id} # Delete biolink + +GET /api/bio/{id}/blocks # List blocks +POST /api/bio/{id}/blocks # Add block +PUT /api/blocks/{id} # Update block +DELETE /api/blocks/{id} # Delete block + +GET /api/bio/{id}/analytics # Summary stats +GET /api/bio/{id}/analytics/geo # Geographic breakdown +GET /api/bio/{id}/analytics/utm # UTM campaign data +``` + +API key routes mirror session routes with `api.auth` and `api.scope.enforce` middleware. + +## MCP Tools + +AI agent tools via Model Context Protocol: + +```php +// Available actions +$actions = [ + 'list', 'get', 'create', 'update', 'delete', + 'add_block', 'update_block', 'delete_block', +]; + +// Example: Create biolink +$response = $bioTools->handle(new Request([ + 'action' => 'create', + 'user_id' => $userId, + 'url' => 'my-page', + 'title' => 'My Page', + 'blocks' => [...], +])); +``` + +Additional MCP tools in separate classes: +- `BioAnalyticsTools` - Analytics queries +- `DomainTools` - Custom domain management +- `PixelTools` - Tracking pixel management +- `ProjectTools` - Project/folder management +- `QrTools` - QR code generation +- `ThemeTools` - Theme management + +## Effects System + +Extensible background effects via `Effects/Catalog.php`: + +```php +// Register effect +Catalog::registerBackgroundEffect('snow', SnowEffect::class); + +// Get effect for rendering +$effect = $page->getBackgroundEffect(); +$html = $effect?->render(); +``` + +Available effects: snow, rain, leaves, autumn_leaves, stars, bubbles, waves, lava_lamp, grid_motion. + +## Multi-Tenancy + +All data is scoped to workspaces using the `BelongsToWorkspace` trait: + +```php +class Page extends Model +{ + use BelongsToWorkspace; // Auto-scopes queries, sets workspace_id on create +} +``` + +The trait: +- Adds global scope to filter by current workspace +- Auto-assigns `workspace_id` on model creation +- Throws `MissingWorkspaceContextException` without valid context + +## Caching Strategy + +- **Domain resolution**: 1-hour cache per domain +- **Public pages**: Cached with `biopage:{domain_id}:{url}` key +- **Analytics**: No caching (queries pre-aggregated ClickStat table) +- **Themes**: System themes cached, user themes not cached + +Cache invalidation triggers: +- Page update clears page cache +- Theme update clears all biolinks using that theme +- Domain update clears domain cache + +## Queue Jobs + +| Job | Purpose | Queue | +|-----|---------|-------| +| `TrackBioLinkClick` | Record individual click with attribution | default | +| `BatchTrackClicks` | Bulk click tracking for high traffic | default | +| `SendBioLinkNotification` | Webhook/email notifications | notifications | +| `SendSubmissionNotification` | Form submission notifications | notifications | + +## Configuration + +The package configuration is merged into Laravel's config as `webpage`: + +```php +// Access config +$defaultDomain = config('webpage.default_domain'); +$blockTypes = config('webpage.block_types'); +$reservedSlugs = config('webpage.reserved_slugs'); +``` + +Key configuration areas: +- `default_domain` - Base domain for biolinks +- `allowed_domains` - Domains that can serve biolinks +- `reserved_slugs` - URLs that cannot be claimed +- `block_types` - All available block types with metadata +- `layout_presets` - HLCRF layout configurations +- `og_images` - Dynamic OG image generation settings +- `revisions` - Revision history limits + +## Database Schema + +Main tables (prefixed `biolink_`): +- `biolinks` - Pages/links +- `biolink_blocks` - Page blocks +- `biolink_themes` - Theme definitions +- `biolink_domains` - Custom domains +- `biolink_projects` - Organisation folders +- `biolink_pixels` - Tracking pixels +- `biolink_clicks` - Individual click records +- `biolink_click_stats` - Aggregated statistics +- `biolink_submissions` - Form submissions +- `biolink_notification_handlers` - Notification configs +- `biolink_pwas` - PWA configurations +- `biolink_push_*` - Push notification tables +- `biolink_templates` - Page templates +- `biolink_revisions` - Change history (separate migration) +- `biolink_edit_locks` - Collaborative editing locks + +## Testing + +Tests use Pest with Orchestra Testbench: + +```bash +# Run all tests +./vendor/bin/pest + +# Run with coverage +./vendor/bin/pest --coverage + +# Run specific test +./vendor/bin/pest --filter=PageTest +``` + +Test categories: +- **Feature tests** - Full integration tests for workflows +- **Unit tests** - Isolated service tests +- **Security tests** - XSS, CSRF, injection prevention +- **Use cases** - Example usage patterns diff --git a/docs/packages/bio/block-types.md b/docs/packages/bio/block-types.md new file mode 100644 index 0000000..1e7b496 --- /dev/null +++ b/docs/packages/bio/block-types.md @@ -0,0 +1,746 @@ +--- +title: Block Types Reference +description: Complete reference for all biolink block types +updated: 2026-01-29 +--- + +# Block Types Reference + +This document provides a complete reference for all available block types in the `core-bio` package. + +## Block Type Categories + +Blocks are organised into four categories: + +| Category | Description | +|----------|-------------| +| `standard` | Basic content blocks (links, text, images) | +| `embeds` | Third-party content embeds (YouTube, Spotify, etc.) | +| `advanced` | Feature-rich blocks (maps, forms, calendars) | +| `payments` | Payment and commerce blocks | + +## Tier Access + +Block types are gated by subscription tier: + +| Tier | Access | +|------|--------| +| `null` (free) | Available to all users | +| `pro` | Requires Pro plan or higher | +| `ultimate` | Requires Ultimate plan | +| `payment` | Requires payment add-on | + +## Standard Blocks + +### link +Basic clickable link button. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-link` | +| Category | standard | +| Has Statistics | Yes | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `url` (string) - Destination URL +- `text` (string) - Button text +- `icon` (string, optional) - FontAwesome icon class + +### heading +Section heading/title. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-heading` | +| Category | standard | +| Has Statistics | No | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `text` (string) - Heading text +- `level` (int) - HTML heading level (1-6) + +### paragraph +Text content block. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-paragraph` | +| Category | standard | +| Has Statistics | No | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `text` (string) - Paragraph content + +### avatar +Profile image display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-user` | +| Category | standard | +| Has Statistics | Yes | +| Themable | No | +| Tier | Free | + +**Settings:** +- `image` (string) - Image path or URL +- `size` (string) - Display size + +### image +Image display block. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-image` | +| Category | standard | +| Has Statistics | Yes | +| Themable | No | +| Tier | Free | + +**Settings:** +- `image` (string) - Image path or URL +- `alt` (string) - Alt text +- `link` (string, optional) - Click destination + +### socials +Social media icon links. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-users` | +| Category | standard | +| Has Statistics | No | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `platforms` (array) - List of platform handles +- `style` (string) - Icon display style + +### business_hours +Opening hours display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-clock` | +| Category | standard | +| Has Statistics | No | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `hours` (array) - Day/time pairs +- `timezone` (string) - Timezone identifier + +### modal_text +Expandable text content. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-book-open` | +| Category | standard | +| Has Statistics | Yes | +| Themable | Yes | +| Tier | Free | + +**Settings:** +- `title` (string) - Modal trigger text +- `content` (string) - Full content + +### header (Pro) +Full-width header section. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-theater-masks` | +| Category | standard | +| Has Statistics | Yes | +| Themable | No | +| Tier | Pro | + +### image_grid (Pro) +Multiple image grid display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-images` | +| Category | standard | +| Has Statistics | Yes | +| Themable | No | +| Tier | Pro | + +### divider (Pro) +Visual separator. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-grip-lines` | +| Category | standard | +| Has Statistics | No | +| Themable | No | +| Tier | Pro | + +### list (Pro) +Bullet/numbered list. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-list` | +| Category | standard | +| Has Statistics | No | +| Themable | No | +| Tier | Pro | + +### big_link (Ultimate) +Large featured link. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-external-link-alt` | +| Category | standard | +| Has Statistics | Yes | +| Themable | Yes | +| Tier | Ultimate | + +### audio (Ultimate) +Audio player. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-volume-up` | +| Category | standard | +| Has Statistics | No | +| Themable | No | +| Tier | Ultimate | + +### video (Ultimate) +Self-hosted video player. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-video` | +| Category | standard | +| Has Statistics | No | +| Themable | No | +| Tier | Ultimate | + +### file (Ultimate) +File download. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-file` | +| Category | standard | +| Has Statistics | Yes | +| Themable | Yes | +| Tier | Ultimate | + +### cta (Ultimate) +Call-to-action block. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-comments` | +| Category | standard | +| Has Statistics | Yes | +| Themable | Yes | +| Tier | Ultimate | + +## Embed Blocks + +### youtube (Free) +YouTube video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.youtube.com, youtu.be | +| Tier | Free | + +**Settings:** +- `url` (string) - YouTube video URL + +### spotify (Free) +Spotify embed (track, album, playlist). + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | open.spotify.com | +| Tier | Free | + +### soundcloud (Free) +SoundCloud track embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | soundcloud.com | +| Tier | Free | + +### tiktok_video (Free) +TikTok video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.tiktok.com | +| Tier | Free | + +### twitch (Free) +Twitch stream/video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.twitch.tv | +| Tier | Free | + +### vimeo (Free) +Vimeo video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | vimeo.com | +| Tier | Free | + +### applemusic (Pro) +Apple Music embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | music.apple.com | +| Tier | Pro | + +### tidal (Pro) +Tidal music embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | tidal.com | +| Tier | Pro | + +### mixcloud (Pro) +Mixcloud embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.mixcloud.com | +| Tier | Pro | + +### kick (Pro) +Kick stream embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | kick.com | +| Tier | Pro | + +### twitter_tweet (Pro) +X/Twitter tweet embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | twitter.com, x.com | +| Tier | Pro | + +### twitter_video (Pro) +X/Twitter video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | twitter.com, x.com | +| Tier | Pro | + +### pinterest_profile (Pro) +Pinterest profile embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | pinterest.com, www.pinterest.com | +| Tier | Pro | + +### instagram_media (Pro) +Instagram post embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.instagram.com | +| Tier | Pro | + +### snapchat (Pro) +Snapchat embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.snapchat.com, snapchat.com | +| Tier | Pro | + +### tiktok_profile (Pro) +TikTok profile embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.tiktok.com | +| Tier | Pro | + +### vk_video (Pro) +VK video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | vk.com | +| Tier | Pro | + +### typeform (Ultimate) +Typeform form embed. + +| Property | Value | +|----------|-------| +| Tier | Ultimate | + +### calendly (Ultimate) +Calendly scheduling embed. + +| Property | Value | +|----------|-------| +| Tier | Ultimate | + +### discord (Ultimate) +Discord server widget. + +| Property | Value | +|----------|-------| +| Tier | Ultimate | + +### facebook (Ultimate) +Facebook content embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.facebook.com, fb.watch | +| Tier | Ultimate | + +### reddit (Ultimate) +Reddit post embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | www.reddit.com | +| Tier | Ultimate | + +### iframe (Ultimate) +Generic iframe embed (use with caution). + +| Property | Value | +|----------|-------| +| Tier | Ultimate | + +### pdf_document (Ultimate) +PDF viewer embed. + +| Property | Value | +|----------|-------| +| Has Statistics | Yes | +| Tier | Ultimate | + +### powerpoint_presentation (Ultimate) +PowerPoint viewer. + +| Property | Value | +|----------|-------| +| Has Statistics | Yes | +| Tier | Ultimate | + +### excel_spreadsheet (Ultimate) +Excel viewer. + +| Property | Value | +|----------|-------| +| Has Statistics | Yes | +| Tier | Ultimate | + +### rumble (Ultimate) +Rumble video embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | rumble.com | +| Tier | Ultimate | + +### telegram (Ultimate) +Telegram channel/post embed. + +| Property | Value | +|----------|-------| +| Whitelisted Hosts | t.me | +| Tier | Ultimate | + +## Advanced Blocks + +### map (Free) +Interactive map display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-map` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Free | + +**Settings:** +- `address` (string) - Location address +- `latitude` (float) - Latitude coordinate +- `longitude` (float) - Longitude coordinate + +### email_collector (Free) +Email signup form. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-envelope` | +| Category | advanced | +| Has Statistics | No | +| Tier | Free | + +**Settings:** +- `placeholder` (string) - Input placeholder +- `button_text` (string) - Submit button text + +### phone_collector (Free) +Phone number collection. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-phone-square-alt` | +| Category | advanced | +| Has Statistics | No | +| Tier | Free | + +### contact_collector (Free) +Full contact form. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-address-book` | +| Category | advanced | +| Has Statistics | No | +| Tier | Free | + +### rss_feed (Pro) +RSS feed display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-rss` | +| Category | advanced | +| Tier | Pro | + +### custom_html (Pro) +Custom HTML content (sanitised). + +| Property | Value | +|----------|-------| +| Icon | `fas fa-code` | +| Category | advanced | +| Tier | Pro | + +### vcard (Pro) +Downloadable contact card. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-id-card` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Pro | + +**Settings:** See `config.vcard_fields` for all available fields. + +### alert (Pro) +Notification/announcement. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-bell` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Pro | + +### appointment_calendar (Ultimate) +Booking/scheduling widget. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-calendar` | +| Category | advanced | +| Tier | Ultimate | + +### faq (Ultimate) +Frequently asked questions. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-feather` | +| Category | advanced | +| Tier | Ultimate | + +### countdown (Ultimate) +Countdown timer. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-clock` | +| Category | advanced | +| Tier | Ultimate | + +### external_item (Ultimate) +External product/item display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-money-bill-wave` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Ultimate | + +### share (Ultimate) +Social sharing buttons. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-share-square` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Ultimate | + +### coupon (Ultimate) +Discount coupon display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-tags` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Ultimate | + +### youtube_feed (Ultimate) +YouTube channel feed. + +| Property | Value | +|----------|-------| +| Icon | `fab fa-youtube` | +| Category | advanced | +| Tier | Ultimate | + +### timeline (Ultimate) +Event timeline display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-ellipsis-v` | +| Category | advanced | +| Tier | Ultimate | + +### review (Ultimate) +Review/testimonial display. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-star` | +| Category | advanced | +| Tier | Ultimate | + +### image_slider (Ultimate) +Image carousel. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-clone` | +| Category | advanced | +| Has Statistics | Yes | +| Tier | Ultimate | + +### markdown (Ultimate) +Markdown content renderer. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-sticky-note` | +| Category | advanced | +| Tier | Ultimate | + +## Payment Blocks + +### paypal (Free) +PayPal payment button. + +| Property | Value | +|----------|-------| +| Icon | `fab fa-paypal` | +| Category | payments | +| Has Statistics | Yes | +| Tier | Free | + +### donation (Payment) +Donation collection. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-hand-holding-usd` | +| Category | payments | +| Tier | Payment add-on | + +### product (Payment) +Product purchase. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-cube` | +| Category | payments | +| Tier | Payment add-on | + +### service (Payment) +Service booking/purchase. + +| Property | Value | +|----------|-------| +| Icon | `fas fa-comments-dollar` | +| Category | payments | +| Tier | Payment add-on | + +## HLCRF Region Support + +Some blocks support placement in multiple layout regions: + +| Block Type | Allowed Regions | +|------------|-----------------| +| link | H, L, C, R, F | +| heading | H, L, C, R, F | +| socials | H, L, C, R, F | +| divider | H, L, C, R, F | + +Blocks without `allowed_regions` config default to Content (C) only. + +## Adding Custom Blocks + +To add a new block type: + +1. Add block definition to `config.php`: +```php +'my_block' => [ + 'icon' => 'fas fa-star', + 'color' => '#ff0000', + 'category' => 'advanced', + 'has_statistics' => true, + 'themable' => true, + 'tier' => 'pro', +], +``` + +2. Create Blade template at `View/Blade/blocks/my_block.blade.php` + +3. Add settings schema validation in relevant request classes + +4. Document the block type in this reference diff --git a/docs/packages/bio/security.md b/docs/packages/bio/security.md new file mode 100644 index 0000000..e9631ac --- /dev/null +++ b/docs/packages/bio/security.md @@ -0,0 +1,438 @@ +--- +title: Security +description: Security considerations and audit notes for core-bio +updated: 2026-01-29 +--- + +# Security Documentation + +This document covers security considerations, threat mitigations, and audit notes for the `core-bio` package. + +## Security Model Overview + +The `core-bio` package handles user-generated content including HTML, CSS, JavaScript, URLs, and file uploads. Security is enforced at multiple layers: + +1. **Input Validation** - Request validation classes +2. **Sanitisation** - Content sanitisers for XSS prevention +3. **Authorisation** - Policies and workspace isolation +4. **Rate Limiting** - Abuse prevention +5. **Output Encoding** - Blade template escaping + +## Authentication and Authorisation + +### Multi-Tenant Isolation + +All data is scoped to workspaces using the `BelongsToWorkspace` trait: + +```php +// Automatic query scoping +$biolinks = Page::all(); // Only returns workspace's biolinks + +// Automatic workspace assignment on create +$biolink = Page::create([...]); // workspace_id auto-set +``` + +Without valid workspace context, `MissingWorkspaceContextException` is thrown. + +### Policy Enforcement + +The `BioPagePolicy` defines access rules: + +| Action | Rule | +|--------|------| +| `viewAny` | Any authenticated user | +| `view` | Owner OR workspace member (read-only) | +| `create` | Has access to at least one workspace | +| `update` | Owner only | +| `delete` | Owner only | + +**Design decision:** Workspace members can view all biolinks within the workspace but only owners can modify. This enables team visibility while protecting individual content. + +### API Authentication + +Two authentication methods: +1. **Session auth** - Standard Laravel session cookies +2. **API key auth** - `Authorization: Bearer hk_xxx` header + +API routes use `api.auth` middleware with scope enforcement (`api.scope.enforce`): +- GET requests require `read` scope +- POST/PUT/PATCH require `write` scope +- DELETE requires `delete` scope + +## XSS Prevention + +### Static Page Sanitisation + +The `StaticPageSanitiser` service handles user-provided HTML/CSS/JS for static pages: + +#### HTML Sanitisation + +Uses HTMLPurifier with strict allowlist: + +**Allowed elements:** +- Structure: div, span, section, article, header, footer, main, nav, aside +- Text: h1-h6, p, br, hr, strong, em, b, i, u, small, mark, del, ins, sub, sup, code, pre, blockquote +- Lists: ul, ol, li, dl, dt, dd +- Links/Media: a, img, picture, source, video, audio, iframe (restricted) +- Tables: table, thead, tbody, tfoot, tr, th, td, caption +- Forms: form, input, textarea, button, label, select, option, fieldset, legend + +**Allowed attributes:** +- Most elements: id, class, style +- Links: href, target, rel +- Images: src, alt, width, height +- iframes: src, width, height (SafeIframe for YouTube/Vimeo only) + +**Blocked completely:** +- `