From 24bafa2d45c2f02f26950fe85815e046836e9ded Mon Sep 17 00:00:00 2001 From: Gurrex <307250828+GurreXr@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:50:38 -0700 Subject: [PATCH] feat(telemetry): persist LLM token usage in the database - Store individual LLM usage records for durable token and call statistics - Expose repository-backed usage summaries by time period and model - Refactor telemetry collection and UI to consume persisted usage data - Update exports and dependency injection for database-backed telemetry - Remove obsolete in-memory persistence and RAG telemetry coupling - Add repository tests for LLM usage persistence --- .../core/telemetry/LlmUsageRepositoryTest.kt | 261 ++++++++++++++++++ .../io/askimo/ui/discover/DiscoverView.kt | 190 +++++++------ .../ui/service/TelemetryExportService.kt | 97 ++----- .../src/main/kotlin/io/askimo/desktop/Main.kt | 13 +- .../io/askimo/desktop/di/DesktopModule.kt | 4 + .../io/askimo/desktop/shell/TelemetryPanel.kt | 209 +++++--------- .../io/askimo/core/context/AppContext.kt | 15 +- .../io/askimo/core/db/DatabaseManager.kt | 40 +++ .../io/askimo/core/rag/RAGContentProcessor.kt | 11 +- .../kotlin/io/askimo/core/rag/RagUtils.kt | 3 - .../askimo/core/telemetry/LlmUsageRecord.kt | 81 ++++++ .../core/telemetry/LlmUsageRepository.kt | 142 ++++++++++ .../core/telemetry/TelemetryCollector.kt | 260 ++++------------- .../telemetry/TelemetryPersistenceManager.kt | 93 ------- 14 files changed, 812 insertions(+), 607 deletions(-) create mode 100644 cli/src/test/kotlin/io/askimo/core/telemetry/LlmUsageRepositoryTest.kt create mode 100644 shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRecord.kt create mode 100644 shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRepository.kt delete mode 100644 shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryPersistenceManager.kt diff --git a/cli/src/test/kotlin/io/askimo/core/telemetry/LlmUsageRepositoryTest.kt b/cli/src/test/kotlin/io/askimo/core/telemetry/LlmUsageRepositoryTest.kt new file mode 100644 index 000000000..2200a74b9 --- /dev/null +++ b/cli/src/test/kotlin/io/askimo/core/telemetry/LlmUsageRepositoryTest.kt @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: AGPLv3 + * + * Copyright (c) 2026 Askimo + */ +package io.askimo.core.telemetry + +import io.askimo.core.db.DatabaseManager +import io.askimo.test.extensions.AskimoTestHome +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import java.time.Instant +import java.time.temporal.ChronoUnit +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * ## LlmUsageRepository Spec + * + * ### insert + * Every LLM call (success or error) is written as an individual row to `llm_usage_records`. + * + * ### queryGroupedByInstance + * Returns per-instance aggregates within a half-open time window `[from, to)`. + * Records are grouped by `COALESCE(instance_id, provider), model` and ordered by total + * tokens descending so the highest-usage model appears first. + */ +@AskimoTestHome +class LlmUsageRepositoryTest { + + private lateinit var db: DatabaseManager + private lateinit var repo: LlmUsageRepository + + @BeforeEach + fun setup() { + db = DatabaseManager.getInMemoryTestInstance(this) + repo = db.getLlmUsageRepository() + } + + @AfterEach + fun tearDown() { + db.close() + DatabaseManager.reset() + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private fun record( + provider: String = "openai", + model: String = "gpt-4o", + instanceId: String? = null, + totalTokens: Int = 100, + durationMs: Long = 500, + isError: Boolean = false, + timestamp: Instant = Instant.now(), + ) = LlmUsageRecord( + provider = provider, + model = model, + instanceId = instanceId, + totalTokens = totalTokens, + durationMs = durationMs, + isError = isError, + timestamp = timestamp, + ) + + /** Inclusive lower bound that captures all records. */ + private val allTime: Instant = Instant.EPOCH + + /** Upper bound well beyond any test record. */ + private val farFuture: Instant = Instant.now().plusSeconds(3_600) + + // ── Insert ──────────────────────────────────────────────────────────────── + + @Nested + inner class Insert { + + @Test + fun `insert does not throw for a successful call`() { + repo.insert(record()) + } + + @Test + fun `insert does not throw for an error call`() { + repo.insert(record(isError = true, totalTokens = 0)) + } + + @Test + fun `inserted record appears in queryGroupedByInstance`() { + repo.insert(record(totalTokens = 200)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(1, stats.size) + assertEquals(200L, stats[0].tokens) + } + } + + // ── queryGroupedByInstance ──────────────────────────────────────────────── + + @Nested + inner class QueryGroupedByInstance { + + @Test + fun `returns empty list when no records exist`() { + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertTrue(stats.isEmpty()) + } + + // ── Time-range filtering ────────────────────────────────────────────── + + @Test + fun `records before the from boundary are excluded`() { + val tooEarly = Instant.now().minus(2, ChronoUnit.HOURS) + val from = Instant.now().minus(1, ChronoUnit.HOURS) + repo.insert(record(timestamp = tooEarly, totalTokens = 999)) + + val stats = repo.queryGroupedByInstance(from, farFuture) + assertTrue(stats.isEmpty()) + } + + @Test + fun `records inside the time window are included`() { + repo.insert(record(timestamp = Instant.now(), totalTokens = 50)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(1, stats.size) + } + + @Test + fun `from is inclusive and to is exclusive`() { + val t0 = Instant.parse("2026-01-01T00:00:00Z") + val t1 = Instant.parse("2026-01-01T01:00:00Z") + val t2 = Instant.parse("2026-01-01T02:00:00Z") + + repo.insert(record(timestamp = t0, totalTokens = 10)) // at lower boundary — included + repo.insert(record(timestamp = t1, totalTokens = 20)) // inside — included + repo.insert(record(timestamp = t2, totalTokens = 30)) // at upper boundary — excluded + + val stats = repo.queryGroupedByInstance(t0, t2) + assertEquals(1, stats.size) + assertEquals(30L, stats[0].tokens) // 10 + 20 + } + + // ── Grouping ────────────────────────────────────────────────────────── + + @Test + fun `calls for the same provider and model are merged into one group`() { + repo.insert(record(provider = "openai", model = "gpt-4o", totalTokens = 100)) + repo.insert(record(provider = "openai", model = "gpt-4o", totalTokens = 200)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(1, stats.size) + assertEquals(300L, stats[0].tokens) + assertEquals(2, stats[0].calls) + } + + @Test + fun `different models under the same provider form separate groups`() { + repo.insert(record(provider = "openai", model = "gpt-4o", totalTokens = 100)) + repo.insert(record(provider = "openai", model = "gpt-3.5-turbo", totalTokens = 50)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(2, stats.size) + } + + @Test + fun `instanceId takes precedence over provider for grouping key`() { + repo.insert(record(provider = "openai", instanceId = "instance-a", model = "gpt-4o", totalTokens = 100)) + repo.insert(record(provider = "openai", instanceId = "instance-a", model = "gpt-4o", totalTokens = 150)) + repo.insert(record(provider = "anthropic", instanceId = "instance-b", model = "claude-3", totalTokens = 80)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(2, stats.size) + assertEquals(250L, stats[0].tokens) // instance-a + assertEquals(80L, stats[1].tokens) // instance-b + } + + // ── instanceKey field ───────────────────────────────────────────────── + + @Test + fun `instanceKey is the provider when instanceId is null`() { + repo.insert(record(provider = "openai", instanceId = null)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals("openai", stats[0].instanceKey) + } + + @Test + fun `instanceKey is the instanceId when present`() { + repo.insert(record(provider = "openai", instanceId = "my-custom-instance")) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals("my-custom-instance", stats[0].instanceKey) + } + + // ── Aggregation ─────────────────────────────────────────────────────── + + @Test + fun `tokens are summed across all calls in a group`() { + repeat(5) { repo.insert(record(totalTokens = 100)) } + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(500L, stats[0].tokens) + } + + @Test + fun `avgDurationMs is the arithmetic mean of all calls in a group`() { + repo.insert(record(durationMs = 200)) + repo.insert(record(durationMs = 400)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(300L, stats[0].avgDurationMs) + } + + @Test + fun `errors counts only the rows with isError true`() { + repo.insert(record(isError = false)) + repo.insert(record(isError = false)) + repo.insert(record(isError = true, totalTokens = 0)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(3, stats[0].calls) + assertEquals(1, stats[0].errors) + } + + @Test + fun `zero errors when all calls succeed`() { + repo.insert(record(isError = false)) + repo.insert(record(isError = false)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(0, stats[0].errors) + } + + // ── Ordering ────────────────────────────────────────────────────────── + + @Test + fun `results are ordered by total tokens descending`() { + repo.insert(record(provider = "openai", model = "cheap", totalTokens = 10)) + repo.insert(record(provider = "anthropic", model = "expensive", totalTokens = 9_000)) + repo.insert(record(provider = "google", model = "medium", totalTokens = 500)) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals(3, stats.size) + assertEquals(9_000L, stats[0].tokens) + assertEquals(500L, stats[1].tokens) + assertEquals(10L, stats[2].tokens) + } + + // ── provider / model fields ─────────────────────────────────────────── + + @Test + fun `provider and model fields are preserved on each stats row`() { + repo.insert(record(provider = "anthropic", model = "claude-3-sonnet")) + + val stats = repo.queryGroupedByInstance(allTime, farFuture) + assertEquals("anthropic", stats[0].provider) + assertEquals("claude-3-sonnet", stats[0].model) + } + } +} diff --git a/desktop-shared/src/main/kotlin/io/askimo/ui/discover/DiscoverView.kt b/desktop-shared/src/main/kotlin/io/askimo/ui/discover/DiscoverView.kt index 5cfb2c047..2ea93dd15 100644 --- a/desktop-shared/src/main/kotlin/io/askimo/ui/discover/DiscoverView.kt +++ b/desktop-shared/src/main/kotlin/io/askimo/ui/discover/DiscoverView.kt @@ -50,6 +50,8 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -66,7 +68,8 @@ import io.askimo.core.AppConstants.DOMAIN import io.askimo.core.chat.domain.ChatSession import io.askimo.core.config.FeatureFlags import io.askimo.core.i18n.LocalizationManager -import io.askimo.core.telemetry.TelemetryMetrics +import io.askimo.core.telemetry.LlmInstanceStats +import io.askimo.core.telemetry.TelemetryCollector import io.askimo.core.user.domain.UserProfile import io.askimo.core.util.TimeUtil import io.askimo.ui.common.components.clickableCard @@ -77,8 +80,11 @@ import io.askimo.ui.common.theme.Spacing import io.askimo.ui.common.theme.ThemePreferences import io.askimo.ui.common.ui.themedTooltip import io.askimo.ui.session.sessionTooltip +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.awt.Desktop import java.net.URI +import java.time.Instant import java.time.LocalTime /** @@ -108,7 +114,7 @@ fun discoverView( showTokenUsageCard: Boolean, onToggleTokenUsageCard: (Boolean) -> Unit, onOpenSystemDiagnostics: () -> Unit, - telemetryMetrics: TelemetryMetrics, + telemetry: TelemetryCollector, modifier: Modifier = Modifier, ) { val scrollState = rememberScrollState() @@ -149,7 +155,7 @@ fun discoverView( if (showTokenUsageCard) { tokenUsageSection( - telemetryMetrics = telemetryMetrics, + telemetry = telemetry, onOpenSystemDiagnostics = onOpenSystemDiagnostics, ) } @@ -358,106 +364,124 @@ private fun statCard( */ @Composable private fun tokenUsageSection( - telemetryMetrics: TelemetryMetrics, + telemetry: TelemetryCollector, onOpenSystemDiagnostics: () -> Unit, ) { - val totalTokens = telemetryMetrics.totalTokensUsed - val topModels = telemetryMetrics.llmTokensByInstance - .entries - .sortedByDescending { it.value } - .take(5) + val refreshSignal by telemetry.refreshSignal.collectAsState() + var stats by remember { mutableStateOf>(emptyList()) } + + LaunchedEffect(refreshSignal) { + stats = withContext(Dispatchers.IO) { + telemetry.usageRepository.queryGroupedByInstance(telemetry.sessionStart, Instant.now()) + } + } + + val totalTokens = stats.sumOf { it.tokens } + val topModels = stats.take(5) // already ordered by tokens DESC from query Column(verticalArrangement = Arrangement.spacedBy(Spacing.medium)) { - // ── Section header ───────────────────────────────────────── + tokenUsageSectionHeader(totalTokens = totalTokens, onOpenSystemDiagnostics = onOpenSystemDiagnostics) + tokenUsageChartCard(totalTokens = totalTokens, topModels = topModels) + } +} + +@Composable +private fun tokenUsageSectionHeader( + totalTokens: Long, + onOpenSystemDiagnostics: () -> Unit, +) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically, + ) { Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.SpaceBetween, + horizontalArrangement = Arrangement.spacedBy(Spacing.small), verticalAlignment = Alignment.CenterVertically, ) { + Icon( + Icons.Default.Token, + contentDescription = null, + modifier = Modifier.size(18.dp), + tint = MaterialTheme.colorScheme.onBackground, + ) + Text( + text = stringResource("discover.tokens.title"), + style = AppTextStyles.sectionTitle, + ) + } + + if (totalTokens > 0) { Row( - horizontalArrangement = Arrangement.spacedBy(Spacing.small), + horizontalArrangement = Arrangement.spacedBy(4.dp), verticalAlignment = Alignment.CenterVertically, ) { - Icon( - Icons.Default.Token, - contentDescription = null, - modifier = Modifier.size(18.dp), - tint = MaterialTheme.colorScheme.onBackground, - ) Text( - text = stringResource("discover.tokens.title"), - style = AppTextStyles.sectionTitle, + // abbreviateTokens uses LocalizationManager internally + text = stringResource("discover.tokens.total", abbreviateTokens(totalTokens)), + style = AppTextStyles.caption, ) - } - - if (totalTokens > 0) { - Row( - horizontalArrangement = Arrangement.spacedBy(4.dp), - verticalAlignment = Alignment.CenterVertically, + IconButton( + onClick = onOpenSystemDiagnostics, + modifier = Modifier.size(24.dp).pointerHoverIcon(PointerIcon.Hand), ) { - Text( - // abbreviateTokens uses LocalizationManager internally - text = stringResource("discover.tokens.total", abbreviateTokens(totalTokens)), - style = AppTextStyles.caption, + Icon( + Icons.AutoMirrored.Filled.OpenInNew, + contentDescription = stringResource("discover.tokens.view_details"), + modifier = Modifier.size(14.dp), + tint = MaterialTheme.colorScheme.onSurfaceVariant, ) - IconButton( - onClick = onOpenSystemDiagnostics, - modifier = Modifier.size(24.dp).pointerHoverIcon(PointerIcon.Hand), - ) { - Icon( - Icons.AutoMirrored.Filled.OpenInNew, - contentDescription = stringResource("discover.tokens.view_details"), - modifier = Modifier.size(14.dp), - tint = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } } } } + } +} - // ── Chart card ───────────────────────────────────────────── - Surface( - shape = MaterialTheme.shapes.large, - tonalElevation = 1.dp, - modifier = Modifier.fillMaxWidth(), - ) { - if (totalTokens == 0L) { - Box( - modifier = Modifier.fillMaxWidth().padding(Spacing.large), - contentAlignment = Alignment.Center, - ) { - Text( - text = stringResource("discover.tokens.empty"), - style = AppTextStyles.bodySecondary, +@Composable +private fun tokenUsageChartCard( + totalTokens: Long, + topModels: List, +) { + Surface( + shape = MaterialTheme.shapes.large, + tonalElevation = 1.dp, + modifier = Modifier.fillMaxWidth(), + ) { + if (totalTokens == 0L) { + Box( + modifier = Modifier.fillMaxWidth().padding(Spacing.large), + contentAlignment = Alignment.Center, + ) { + Text( + text = stringResource("discover.tokens.empty"), + style = AppTextStyles.bodySecondary, + ) + } + } else { + Column( + modifier = Modifier.fillMaxWidth().padding(Spacing.large), + verticalArrangement = Arrangement.spacedBy(Spacing.medium), + ) { + topModels.forEachIndexed { index, stat -> + val provider = stat.instanceKey + .split(":", limit = 2).getOrElse(0) { stat.instanceKey } + .replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() } + val model = stat.model.ifBlank { provider } + val fraction = stat.tokens.toFloat() / totalTokens.toFloat() + val pctFormatted = LocalizationManager.formatNumber((fraction * 100).toInt()) + "%" + + tokenBarRow( + provider = provider, + model = model, + tokens = stat.tokens, + fraction = fraction, + pctFormatted = pctFormatted, ) - } - } else { - Column( - modifier = Modifier.fillMaxWidth().padding(Spacing.large), - verticalArrangement = Arrangement.spacedBy(Spacing.medium), - ) { - topModels.forEachIndexed { index, (key, tokens) -> - val parts = key.split(":", limit = 2) - val provider = parts.getOrElse(0) { key } - .replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() } - val model = parts.getOrElse(1) { "" }.ifBlank { provider } - val fraction = tokens.toFloat() / totalTokens.toFloat() - // Percentage: locale-aware integer, e.g. "62 %" in fr vs "62%" in en - val pctFormatted = LocalizationManager.formatNumber((fraction * 100).toInt()) + "%" - - tokenBarRow( - provider = provider, - model = model, - tokens = tokens, - fraction = fraction, - pctFormatted = pctFormatted, - ) - if (index < topModels.lastIndex) { - HorizontalDivider( - color = MaterialTheme.colorScheme.outlineVariant.copy(alpha = 0.3f), - ) - } + if (index < topModels.lastIndex) { + HorizontalDivider( + color = MaterialTheme.colorScheme.outlineVariant.copy(alpha = 0.3f), + ) } } } diff --git a/desktop-shared/src/main/kotlin/io/askimo/ui/service/TelemetryExportService.kt b/desktop-shared/src/main/kotlin/io/askimo/ui/service/TelemetryExportService.kt index 33d0afe95..17b243139 100644 --- a/desktop-shared/src/main/kotlin/io/askimo/ui/service/TelemetryExportService.kt +++ b/desktop-shared/src/main/kotlin/io/askimo/ui/service/TelemetryExportService.kt @@ -6,7 +6,8 @@ package io.askimo.ui.service import io.askimo.core.i18n.LocalizationManager import io.askimo.core.logging.logger -import io.askimo.core.telemetry.TelemetryMetrics +import io.askimo.core.telemetry.LlmInstanceStats +import io.askimo.core.telemetry.TelemetryCollector import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.io.File @@ -18,18 +19,13 @@ import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream /** - * Exports a [TelemetryMetrics] snapshot to a ZIP file containing two CSVs: + * Exports LLM usage metrics for the current session to a ZIP file containing two CSVs: * - * - `session-metrics.csv` — aggregated RAG and LLM summary metrics. + * - `session-metrics.csv` — aggregated LLM summary metrics. * - `model-token-usage.csv` — per-provider/model LLM call breakdown. * - * CSV conventions: - * - UTF-8 encoding, comma delimiter, RFC 4180 quoting (all fields quoted). - * - English-only stable column headers for downstream tooling compatibility. - * - Numeric values formatted via [LocalizationManager] using the current user locale. - * - Decimal values use [LocalizationManager.formatDouble] with 2 fraction digits. - * - Empty rows are written (header only) when a section has no data. - * - Timestamps are ISO-8601 UTC. + * Data is queried from [TelemetryCollector.usageRepository] scoped to + * [[TelemetryCollector.sessionStart], now). */ object TelemetryExportService { @@ -38,26 +34,27 @@ object TelemetryExportService { .withZone(ZoneOffset.UTC) /** - * Exports [metrics] to [targetZipFile] as a ZIP with two CSV entries. + * Exports session metrics to [targetZipFile] as a ZIP with two CSV entries. * - * @param metrics Current telemetry snapshot (may have all-zero values). + * @param telemetry Current [TelemetryCollector] (provides sessionStart + repository). * @param targetZipFile Destination file; parent directories are created if needed. * @return [Result.success] on completion, [Result.failure] on any error. */ - suspend fun export(metrics: TelemetryMetrics, targetZipFile: File): Result = withContext(Dispatchers.IO) { + suspend fun export(telemetry: TelemetryCollector, targetZipFile: File): Result = withContext(Dispatchers.IO) { runCatching { targetZipFile.parentFile?.mkdirs() val capturedAt = timestampFormatter.format(Instant.now()) + val stats = telemetry.usageRepository.queryGroupedByInstance(telemetry.sessionStart, Instant.now()) ZipOutputStream(targetZipFile.outputStream().buffered()).use { zos -> // ── session-metrics.csv ────────────────────────────────── zos.putNextEntry(ZipEntry("session-metrics.csv")) - zos.write(buildSessionMetricsCsv(metrics, capturedAt).toByteArray(Charsets.UTF_8)) + zos.write(buildSessionMetricsCsv(stats, capturedAt).toByteArray(Charsets.UTF_8)) zos.closeEntry() // ── model-token-usage.csv ──────────────────────────────── zos.putNextEntry(ZipEntry("model-token-usage.csv")) - zos.write(buildModelTokenUsageCsv(metrics, capturedAt).toByteArray(Charsets.UTF_8)) + zos.write(buildModelTokenUsageCsv(stats, capturedAt).toByteArray(Charsets.UTF_8)) zos.closeEntry() } @@ -69,36 +66,14 @@ object TelemetryExportService { // ── CSV builders ───────────────────────────────────────────────────────── - private fun buildSessionMetricsCsv(metrics: TelemetryMetrics, capturedAt: String): String { + private fun buildSessionMetricsCsv(stats: List, capturedAt: String): String { val sw = StringWriter() - sw.appendCsvLine( - "captured_at", - "metric_key", - "metric_label", - "unit", - "value", - ) - - // RAG metrics — only if any RAG data was collected - if (metrics.ragClassificationTotal > 0) { - sw.appendCsvLine(capturedAt, "rag_classification_total", "RAG Classification Total", "count", LocalizationManager.formatNumber(metrics.ragClassificationTotal)) - sw.appendCsvLine(capturedAt, "rag_triggered", "RAG Triggered", "count", LocalizationManager.formatNumber(metrics.ragTriggered)) - sw.appendCsvLine(capturedAt, "rag_skipped", "RAG Skipped", "count", LocalizationManager.formatNumber(metrics.ragSkipped)) - sw.appendCsvLine(capturedAt, "rag_triggered_percent", "RAG Triggered Percent", "%", LocalizationManager.formatDouble(metrics.ragTriggeredPercent, 2)) - sw.appendCsvLine(capturedAt, "rag_avg_classification_time_ms", "RAG Avg Classification Time", "ms", LocalizationManager.formatNumber(metrics.ragAvgClassificationTimeMs)) - } - - if (metrics.ragRetrievalTotal > 0) { - sw.appendCsvLine(capturedAt, "rag_retrieval_total", "RAG Retrieval Total", "count", LocalizationManager.formatNumber(metrics.ragRetrievalTotal)) - sw.appendCsvLine(capturedAt, "rag_avg_retrieval_time_ms", "RAG Avg Retrieval Time", "ms", LocalizationManager.formatNumber(metrics.ragAvgRetrievalTimeMs)) - sw.appendCsvLine(capturedAt, "rag_avg_chunks_retrieved", "RAG Avg Chunks Retrieved", "count", LocalizationManager.formatDouble(metrics.ragAvgChunksRetrieved, 2)) - } + sw.appendCsvLine("captured_at", "metric_key", "metric_label", "unit", "value") - // LLM summary metrics - if (metrics.llmCallsByInstance.isNotEmpty()) { - val totalCalls = metrics.llmCallsByInstance.values.sum() - val totalTokens = metrics.llmTokensByInstance.values.sum() - val totalErrors = metrics.llmErrorsByInstance.values.sum() + if (stats.isNotEmpty()) { + val totalCalls = stats.sumOf { it.calls } + val totalTokens = stats.sumOf { it.tokens } + val totalErrors = stats.sumOf { it.errors } sw.appendCsvLine(capturedAt, "llm_total_calls", "LLM Total Calls", "count", LocalizationManager.formatNumber(totalCalls)) sw.appendCsvLine(capturedAt, "llm_total_tokens", "LLM Total Tokens", "tokens", LocalizationManager.formatNumber(totalTokens)) sw.appendCsvLine(capturedAt, "llm_total_errors", "LLM Total Errors", "count", LocalizationManager.formatNumber(totalErrors)) @@ -107,34 +82,20 @@ object TelemetryExportService { return sw.toString() } - private fun buildModelTokenUsageCsv(metrics: TelemetryMetrics, capturedAt: String): String { + private fun buildModelTokenUsageCsv(stats: List, capturedAt: String): String { val sw = StringWriter() - sw.appendCsvLine( - "captured_at", - "instance_or_provider", - "model", - "calls", - "tokens", - "avg_duration_ms", - "errors", - ) - - metrics.llmCallsByInstance.forEach { (instanceModel, calls) -> - val parts = instanceModel.split(":", limit = 2) - val instanceOrProvider = parts.getOrElse(0) { instanceModel } - val model = parts.getOrElse(1) { "" } - val tokens = metrics.llmTokensByInstance[instanceModel] ?: 0L - val avgDurationMs = metrics.llmAvgDurationMsByInstance[instanceModel] ?: 0L - val errors = metrics.llmErrorsByInstance[instanceModel] ?: 0 + sw.appendCsvLine("captured_at", "instance_or_provider", "model", "calls", "tokens", "avg_duration_ms", "errors") + stats.forEach { stat -> + val instanceOrProvider = stat.instanceKey.split(":", limit = 2).getOrElse(0) { stat.instanceKey } sw.appendCsvLine( capturedAt, instanceOrProvider, - model, - LocalizationManager.formatNumber(calls), - LocalizationManager.formatNumber(tokens), - LocalizationManager.formatNumber(avgDurationMs), - LocalizationManager.formatNumber(errors), + stat.model, + LocalizationManager.formatNumber(stat.calls), + LocalizationManager.formatNumber(stat.tokens), + LocalizationManager.formatNumber(stat.avgDurationMs), + LocalizationManager.formatNumber(stat.errors), ) } @@ -143,10 +104,6 @@ object TelemetryExportService { // ── CSV helpers ─────────────────────────────────────────────────────────── - /** - * Appends a RFC 4180 CSV row: all fields are quoted, internal quotes are doubled, - * newlines within values are escaped as \n. - */ private fun StringWriter.appendCsvLine(vararg fields: Any) { append( fields.joinToString(",") { field -> diff --git a/desktop/src/main/kotlin/io/askimo/desktop/Main.kt b/desktop/src/main/kotlin/io/askimo/desktop/Main.kt index 56954f1ec..f53c87a85 100644 --- a/desktop/src/main/kotlin/io/askimo/desktop/Main.kt +++ b/desktop/src/main/kotlin/io/askimo/desktop/Main.kt @@ -185,6 +185,7 @@ import java.awt.Toolkit import java.net.URI import java.nio.file.Path import java.nio.file.Paths +import java.time.Instant import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.Locale @@ -292,7 +293,8 @@ fun main(args: Array) { icon = icon, onCloseRequest = { val messageCount = runCatching { - AppContext.getInstance().telemetry.metricsFlow.value.llmCallsByInstance.values.sum() + val t = AppContext.getInstance().telemetry + t.usageRepository.countByPeriod(t.sessionStart, Instant.now()) }.getOrDefault(0) Analytics.trackSessionEnd(messageCount) Analytics.shutdown() @@ -1489,7 +1491,6 @@ fun app(frameWindowScope: FrameWindowScope? = null, windowState: WindowState? = // System Diagnostics Dialog if (showSystemDiagnosticsDialog) { - val metrics by appContext.telemetry.metricsFlow.collectAsState() systemResourcesDialog( onDismiss = { showSystemDiagnosticsDialog = false }, onExportTelemetry = { @@ -1503,7 +1504,7 @@ fun app(frameWindowScope: FrameWindowScope? = null, windowState: WindowState? = title = LocalizationManager.getString("telemetry.export.dialog.title"), ) ?: return@launch - val result = TelemetryExportService.export(metrics, targetFile) + val result = TelemetryExportService.export(appContext.telemetry, targetFile) if (result.isFailure) { errorDialogState = ErrorDialogState( show = true, @@ -1516,7 +1517,7 @@ fun app(frameWindowScope: FrameWindowScope? = null, windowState: WindowState? = } } }, - telemetryContent = { telemetryPanel(metrics = metrics, maxHeight = 480.dp) }, + telemetryContent = { telemetryPanel(maxHeight = 480.dp) }, ) } @@ -2016,7 +2017,7 @@ fun mainContent( onOpenSystemDiagnostics: () -> Unit = {}, bookmarksViewModel: BookmarksViewModel? = null, ) { - val discoverMetrics by appContext.telemetry.metricsFlow.collectAsState() + val discoverRefresh by appContext.telemetry.refreshSignal.collectAsState() Box( modifier = Modifier .fillMaxSize() @@ -2043,7 +2044,7 @@ fun mainContent( showTokenUsageCard = showTokenUsageCard, onToggleTokenUsageCard = onToggleTokenUsageCard, onOpenSystemDiagnostics = onOpenSystemDiagnostics, - telemetryMetrics = discoverMetrics, + telemetry = appContext.telemetry, modifier = Modifier.fillMaxSize(), ) diff --git a/desktop/src/main/kotlin/io/askimo/desktop/di/DesktopModule.kt b/desktop/src/main/kotlin/io/askimo/desktop/di/DesktopModule.kt index 611e78e49..061cc51ba 100644 --- a/desktop/src/main/kotlin/io/askimo/desktop/di/DesktopModule.kt +++ b/desktop/src/main/kotlin/io/askimo/desktop/di/DesktopModule.kt @@ -14,6 +14,7 @@ import io.askimo.core.mcp.McpInstanceService import io.askimo.core.plan.PlanService import io.askimo.core.plan.repository.PlanDefRepository import io.askimo.core.providers.ProviderInstanceService +import io.askimo.core.telemetry.TelemetryCollector import io.askimo.core.tools.ToolProviderImpl import io.askimo.desktop.project.ProjectViewModel import io.askimo.desktop.project.ProjectsViewModel @@ -49,6 +50,9 @@ val desktopModule = module { single { get().getChatDirectiveRepository() } single { get().getProjectRepository() } single { get().getPlanExecutionRepository() } + single { get().getLlmUsageRepository() } + + single { TelemetryCollector(usageRepository = get()) } single { ProjectService(projectRepository = get()) } diff --git a/desktop/src/main/kotlin/io/askimo/desktop/shell/TelemetryPanel.kt b/desktop/src/main/kotlin/io/askimo/desktop/shell/TelemetryPanel.kt index 4813bad1b..e72445bd1 100644 --- a/desktop/src/main/kotlin/io/askimo/desktop/shell/TelemetryPanel.kt +++ b/desktop/src/main/kotlin/io/askimo/desktop/shell/TelemetryPanel.kt @@ -31,6 +31,8 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -46,7 +48,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import io.askimo.core.context.AppContext import io.askimo.core.i18n.LocalizationManager -import io.askimo.core.telemetry.TelemetryMetrics +import io.askimo.core.telemetry.LlmInstanceStats import io.askimo.ui.common.i18n.stringResource import io.askimo.ui.common.theme.AppComponents import io.askimo.ui.common.theme.AppTextStyles @@ -54,7 +56,10 @@ import io.askimo.ui.common.theme.Spacing import io.askimo.ui.common.ui.themedTooltip import io.askimo.ui.util.formatDuration import io.askimo.ui.util.formatDurationDetailed +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.koin.java.KoinJavaComponent.get +import java.time.Instant import java.util.Locale.getDefault /** @@ -62,8 +67,17 @@ import java.util.Locale.getDefault * Max height is limited to 1/3 of parent height with scrolling support. */ @Composable -internal fun telemetryPanel(metrics: TelemetryMetrics, maxHeight: Dp) { +internal fun telemetryPanel(maxHeight: Dp) { val appContext = remember { get(AppContext::class.java) } + val telemetry = appContext.telemetry + val refreshSignal by telemetry.refreshSignal.collectAsState() + var stats by remember { mutableStateOf>(emptyList()) } + + LaunchedEffect(refreshSignal) { + stats = withContext(Dispatchers.IO) { + telemetry.usageRepository.queryGroupedByInstance(telemetry.sessionStart, Instant.now()) + } + } Surface( modifier = Modifier.fillMaxWidth(), @@ -95,10 +109,10 @@ internal fun telemetryPanel(metrics: TelemetryMetrics, maxHeight: Dp) { style = AppTextStyles.itemTitle, ) - if (metrics.ragClassificationTotal > 0 || metrics.llmCallsByInstance.isNotEmpty()) { + if (stats.isNotEmpty()) { themedTooltip(text = stringResource("telemetry.reset")) { IconButton( - onClick = { appContext.telemetry.reset() }, + onClick = { telemetry.reset() }, modifier = Modifier.size(24.dp), ) { Icon( @@ -112,7 +126,7 @@ internal fun telemetryPanel(metrics: TelemetryMetrics, maxHeight: Dp) { } } - if (metrics.ragClassificationTotal == 0 && metrics.llmCallsByInstance.isEmpty()) { + if (stats.isEmpty()) { Text( text = stringResource("telemetry.no.data"), style = AppTextStyles.bodySecondary, @@ -121,145 +135,72 @@ internal fun telemetryPanel(metrics: TelemetryMetrics, maxHeight: Dp) { return@Column } - // ── RAG section ────────────────────────────────────────── - if (metrics.ragClassificationTotal > 0) { - Text( - text = stringResource("telemetry.tab.rag"), - style = AppTextStyles.fieldLabel, - fontWeight = FontWeight.SemiBold, - ) - - // Summary stats - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.spacedBy(Spacing.small), - ) { - telemetryStat( - label = stringResource("telemetry.rag.total.queries"), - value = LocalizationManager.formatNumber(metrics.ragClassificationTotal), - modifier = Modifier.weight(1f), - ) - telemetryStat( - label = stringResource("telemetry.rag.triggered.label"), - value = "${LocalizationManager.formatNumber(metrics.ragTriggered)} (${LocalizationManager.formatDouble(metrics.ragTriggeredPercent, 0)}%)", - modifier = Modifier.weight(1f), - ) - telemetryStat( - label = stringResource("telemetry.rag.skipped.label"), - value = LocalizationManager.formatNumber(metrics.ragSkipped), - modifier = Modifier.weight(1f), - ) - } - - // Detail cards - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.spacedBy(Spacing.medium), - ) { - telemetryMetricCard( - label = stringResource("telemetry.rag.efficiency"), - value = "${LocalizationManager.formatDouble(metrics.ragTriggeredPercent, 0)}%", - subtitle = stringResource("telemetry.rag.triggered", metrics.ragTriggered, metrics.ragClassificationTotal), - modifier = Modifier.weight(1f), - ) - telemetryMetricCard( - label = stringResource("telemetry.classification"), - value = formatDuration(metrics.ragAvgClassificationTimeMs), - valueTooltip = formatDurationDetailed(metrics.ragAvgClassificationTimeMs), - subtitle = stringResource("telemetry.classification.time"), - modifier = Modifier.weight(1f), - ) - if (metrics.ragRetrievalTotal > 0) { - telemetryMetricCard( - label = stringResource("telemetry.retrieval"), - value = formatDuration(metrics.ragAvgRetrievalTimeMs), - valueTooltip = formatDurationDetailed(metrics.ragAvgRetrievalTimeMs), - subtitle = stringResource("telemetry.retrieval.chunks", LocalizationManager.formatDouble(metrics.ragAvgChunksRetrieved, 1)), - modifier = Modifier.weight(1f), - ) - } - } - } - // ── LLM section ─────────────────────────────────────────── - if (metrics.llmCallsByInstance.isNotEmpty()) { - if (metrics.ragClassificationTotal > 0) { - HorizontalDivider() - } - - Text( - text = stringResource("telemetry.tab.llm"), - style = AppTextStyles.fieldLabel, - fontWeight = FontWeight.SemiBold, - ) - - var sortColumn by remember { mutableStateOf(LlmSortColumn.INSTANCE) } - var sortAscending by remember { mutableStateOf(true) } - - fun toggleSort(column: LlmSortColumn) { - if (sortColumn == column) { - sortAscending = !sortAscending - } else { - sortColumn = column - sortAscending = true - } - } - - // Table header - llmTableHeader( - sortColumn = sortColumn, - sortAscending = sortAscending, - onSort = ::toggleSort, - ) - - HorizontalDivider() + Text( + text = stringResource("telemetry.tab.llm"), + style = AppTextStyles.fieldLabel, + fontWeight = FontWeight.SemiBold, + ) - var totalCalls = 0 - var totalTokens = 0L - var totalErrors = 0 + var sortColumn by remember { mutableStateOf(LlmSortColumn.INSTANCE) } + var sortAscending by remember { mutableStateOf(true) } - val rows = metrics.llmCallsByInstance.map { (providerModel, calls) -> - val parts = providerModel.split(":", limit = 2) - val instance = parts.getOrElse(0) { providerModel } - .replaceFirstChar { if (it.isLowerCase()) it.titlecase(getDefault()) else it.toString() } - val model = parts.getOrElse(1) { "" } - val tokens = metrics.llmTokensByInstance[providerModel] ?: 0L - val avgDuration = metrics.llmAvgDurationMsByInstance[providerModel] ?: 0L - val errors = metrics.llmErrorsByInstance[providerModel] ?: 0 - LlmRow(instance, model, calls, tokens, avgDuration, errors) + fun toggleSort(column: LlmSortColumn) { + if (sortColumn == column) { + sortAscending = !sortAscending + } else { + sortColumn = column + sortAscending = true } + } - val sorted = when (sortColumn) { - LlmSortColumn.INSTANCE -> rows.sortedBy { it.instance } - LlmSortColumn.MODEL -> rows.sortedBy { it.model } - LlmSortColumn.CALLS -> rows.sortedBy { it.calls } - LlmSortColumn.TOKENS -> rows.sortedBy { it.tokens } - LlmSortColumn.AVG_DURATION -> rows.sortedBy { it.avgDurationMs } - LlmSortColumn.ERRORS -> rows.sortedBy { it.errors } - }.let { if (sortAscending) it else it.reversed() } + llmTableHeader( + sortColumn = sortColumn, + sortAscending = sortAscending, + onSort = ::toggleSort, + ) - sorted.forEach { row -> - totalCalls += row.calls - totalTokens += row.tokens - totalErrors += row.errors + HorizontalDivider() - llmTableDataRow(row) - } + var totalCalls = 0 + var totalTokens = 0L + var totalErrors = 0 - HorizontalDivider() + val rows = stats.map { stat -> + val instance = stat.instanceKey + .split(":", limit = 2).getOrElse(0) { stat.instanceKey } + .replaceFirstChar { if (it.isLowerCase()) it.titlecase(getDefault()) else it.toString() } + LlmRow(instance, stat.model, stat.calls, stat.tokens, stat.avgDurationMs, stat.errors) + } - // Totals row - llmTableRow( - instance = stringResource("telemetry.llm.col.total"), - model = "", - calls = LocalizationManager.formatNumber(totalCalls), - tokens = LocalizationManager.formatNumber(totalTokens), - avgDuration = "", - errors = if (totalErrors > 0) LocalizationManager.formatNumber(totalErrors) else "—", - isHeader = true, - errorsIsError = totalErrors > 0, - ) + val sorted = when (sortColumn) { + LlmSortColumn.INSTANCE -> rows.sortedBy { it.instance } + LlmSortColumn.MODEL -> rows.sortedBy { it.model } + LlmSortColumn.CALLS -> rows.sortedBy { it.calls } + LlmSortColumn.TOKENS -> rows.sortedBy { it.tokens } + LlmSortColumn.AVG_DURATION -> rows.sortedBy { it.avgDurationMs } + LlmSortColumn.ERRORS -> rows.sortedBy { it.errors } + }.let { if (sortAscending) it else it.reversed() } + + sorted.forEach { row -> + totalCalls += row.calls + totalTokens += row.tokens + totalErrors += row.errors + llmTableDataRow(row) } + + HorizontalDivider() + + llmTableRow( + instance = stringResource("telemetry.llm.col.total"), + model = "", + calls = LocalizationManager.formatNumber(totalCalls), + tokens = LocalizationManager.formatNumber(totalTokens), + avgDuration = "", + errors = if (totalErrors > 0) LocalizationManager.formatNumber(totalErrors) else "—", + isHeader = true, + errorsIsError = totalErrors > 0, + ) } VerticalScrollbar( diff --git a/shared/src/main/kotlin/io/askimo/core/context/AppContext.kt b/shared/src/main/kotlin/io/askimo/core/context/AppContext.kt index bfe6cb2dd..aaf042a99 100644 --- a/shared/src/main/kotlin/io/askimo/core/context/AppContext.kt +++ b/shared/src/main/kotlin/io/askimo/core/context/AppContext.kt @@ -119,10 +119,19 @@ class AppContext private constructor( get() = _userProfileDirective /** - * Telemetry collector for tracking RAG and LLM metrics. + * Telemetry collector for tracking LLM call metrics. * Shared across all chat clients in this context. - */ - val telemetry = TelemetryCollector() + * + * In desktop mode, [TelemetryCollector] is resolved from the Koin graph (where + * [io.askimo.core.telemetry.LlmUsageRepository] has already been injected). + * In CLI / stateless mode (Koin not started), falls back to constructing it + * directly with a [DatabaseManager] singleton. + */ + val telemetry: TelemetryCollector = runCatching { + getKoin().get() + }.getOrElse { + TelemetryCollector(usageRepository = DatabaseManager.getInstance().getLlmUsageRepository()) + } /** * Cached utility client for lightweight operations (classification, intent detection). diff --git a/shared/src/main/kotlin/io/askimo/core/db/DatabaseManager.kt b/shared/src/main/kotlin/io/askimo/core/db/DatabaseManager.kt index 9c421266d..ef90f2349 100644 --- a/shared/src/main/kotlin/io/askimo/core/db/DatabaseManager.kt +++ b/shared/src/main/kotlin/io/askimo/core/db/DatabaseManager.kt @@ -17,6 +17,7 @@ import io.askimo.core.chat.repository.SessionMemoryRepository import io.askimo.core.chat.repository.UserMemoryRepository import io.askimo.core.plan.repository.PlanExecutionRepository import io.askimo.core.skills.repository.SkillRunHistoryRepository +import io.askimo.core.telemetry.LlmUsageRepository import io.askimo.core.user.repository.UserProfileRepository import io.askimo.core.util.AskimoHome import java.sql.Connection @@ -112,6 +113,7 @@ class DatabaseManager private constructor( createIndexFileStateTable(connection) createPlanExecutionsTable(connection) createSkillRunHistoryTable(connection) + createLlmUsageRecordsTable(connection) } private fun createUserProfilesTable(conn: Connection) { @@ -737,6 +739,34 @@ class DatabaseManager private constructor( } } + private fun createLlmUsageRecordsTable(conn: Connection) { + conn.createStatement().use { stmt -> + stmt.executeUpdate( + """ + CREATE TABLE IF NOT EXISTS llm_usage_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + instance_id TEXT, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + duration_ms INTEGER NOT NULL DEFAULT 0, + is_error INTEGER NOT NULL DEFAULT 0 + ) + """.trimIndent(), + ) + + stmt.executeUpdate( + """ + CREATE INDEX IF NOT EXISTS idx_llm_usage_records_timestamp + ON llm_usage_records (timestamp) + """.trimIndent(), + ) + } + } + private val _chatSessionRepository: ChatSessionRepository by lazy { ChatSessionRepository(this) } @@ -785,6 +815,10 @@ class DatabaseManager private constructor( SkillRunHistoryRepository(this) } + private val _llmUsageRepository: LlmUsageRepository by lazy { + LlmUsageRepository(this) + } + /** * Get the singleton ChatSessionRepository instance. * All access to chat sessions should go through this repository. @@ -857,6 +891,12 @@ class DatabaseManager private constructor( */ fun getSkillRunHistoryRepository(): SkillRunHistoryRepository = _skillRunHistoryRepository + /** + * Get the singleton LlmUsageRepository instance. + * All access to individual LLM call records should go through this repository. + */ + fun getLlmUsageRepository(): LlmUsageRepository = _llmUsageRepository + /** * Get the singleton FileSegmentRepository instance (deprecated - use getResourceSegmentRepository). * All access to file-segment mappings should go through this repository. diff --git a/shared/src/main/kotlin/io/askimo/core/rag/RAGContentProcessor.kt b/shared/src/main/kotlin/io/askimo/core/rag/RAGContentProcessor.kt index b07402657..a35d04752 100644 --- a/shared/src/main/kotlin/io/askimo/core/rag/RAGContentProcessor.kt +++ b/shared/src/main/kotlin/io/askimo/core/rag/RAGContentProcessor.kt @@ -10,7 +10,6 @@ import dev.langchain4j.rag.content.retriever.ContentRetriever import dev.langchain4j.rag.query.Query import io.askimo.core.logging.logger import io.askimo.core.providers.ChatClient -import io.askimo.core.telemetry.TelemetryCollector import io.askimo.core.util.ProcessBuilderExt import kotlinx.coroutines.runBlocking import java.io.File @@ -25,12 +24,10 @@ import java.io.File * * @property delegate The wrapped semantic retriever (HybridContentRetriever). * @property knowledgeSourcePaths Root paths of the project's knowledge sources, used for grep search. - * @property telemetry Optional telemetry collector. */ class RAGContentProcessor( private val delegate: ContentRetriever, private val classifierChatClient: ChatClient, - private val telemetry: TelemetryCollector? = null, private val knowledgeSourcePaths: List = emptyList(), ) : ContentRetriever { @@ -58,16 +55,12 @@ class RAGContentProcessor( classifier.classify(userMessage, conversationHistory, knowledgeSourcePaths) } val classificationDuration = System.currentTimeMillis() - classificationStartTime - - telemetry?.recordRAGClassification(intent != RAGIntent.SKIP, classificationDuration) + log.debug("RAG classification took ${classificationDuration}ms, intent=$intent") return when (intent) { RAGIntent.RAG -> { log.info("RAG triggered — semantic retrieval") - val retrievalStartTime = System.currentTimeMillis() - val results = delegate.retrieve(query) - telemetry?.recordRAGRetrieval(results.size, System.currentTimeMillis() - retrievalStartTime) - results + delegate.retrieve(query) } RAGIntent.SEARCH -> { diff --git a/shared/src/main/kotlin/io/askimo/core/rag/RagUtils.kt b/shared/src/main/kotlin/io/askimo/core/rag/RagUtils.kt index e38f5cfc3..48b1e0a0c 100644 --- a/shared/src/main/kotlin/io/askimo/core/rag/RagUtils.kt +++ b/shared/src/main/kotlin/io/askimo/core/rag/RagUtils.kt @@ -10,7 +10,6 @@ import dev.langchain4j.model.embedding.EmbeddingModel import dev.langchain4j.rag.content.retriever.ContentRetriever import dev.langchain4j.store.embedding.EmbeddingStore import io.askimo.core.config.AppConfig -import io.askimo.core.context.AppContext import io.askimo.core.logging.logger import io.askimo.core.providers.ChatClient import io.askimo.core.util.AskimoHome @@ -114,7 +113,6 @@ object RagUtils { useWebSearch: Boolean = false, ): ContentRetriever { val ragConfig = AppConfig.rag - val telemetry = AppContext.getInstance().telemetry val webRetriever: ContentRetriever? = if (useWebSearch && AppConfig.webSearch.enabled) { val backend = WebSearchDispatcher.activeBackend(AppConfig.webSearch) @@ -133,7 +131,6 @@ object RagUtils { webRetriever = webRetriever, ), classifierChatClient, - telemetry, knowledgeSourcePaths, ) } diff --git a/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRecord.kt b/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRecord.kt new file mode 100644 index 000000000..d5a8f15df --- /dev/null +++ b/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRecord.kt @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: AGPLv3 + * + * Copyright (c) 2026 Askimo + */ +package io.askimo.core.telemetry + +import io.askimo.core.db.sqliteInstant +import org.jetbrains.exposed.v1.core.Table +import java.time.Instant + +/** + * Persisted record of a single LLM call (success or error). + * + * @param id Auto-generated primary key (AUTOINCREMENT, 0 = not yet persisted). + * @param timestamp When the call completed. + * @param provider Provider identifier (e.g. "openai", "anthropic"). + * @param model Model name as reported by the provider. + * @param instanceId Optional instance key — "$instanceId:$model" composite used by TelemetryCollector. + * @param promptTokens Input token count (0 when unknown). + * @param outputTokens Output/completion token count (0 when unknown). + * @param totalTokens Combined token count (0 when unknown). + * @param durationMs Wall-clock duration of the call in milliseconds. + * @param isError true when the call ended with an error instead of a response. + */ +data class LlmUsageRecord( + val id: Long = 0, + val timestamp: Instant = Instant.now(), + val provider: String, + val model: String, + val instanceId: String? = null, + val promptTokens: Int = 0, + val outputTokens: Int = 0, + val totalTokens: Int = 0, + val durationMs: Long = 0, + val isError: Boolean = false, +) + +/** + * Exposed table definition for llm_usage_records. + * + * The [timestamp] column uses [sqliteInstant] (ISO-8601 TEXT) consistent with all + * other timestamp columns in the schema, and is indexed for efficient range queries. + */ +object LlmUsageRecordTable : Table("llm_usage_records") { + val id = long("id").autoIncrement() + val timestamp = sqliteInstant("timestamp") + val provider = text("provider") + val model = text("model") + val instanceId = text("instance_id").nullable() + val promptTokens = integer("prompt_tokens").default(0) + val outputTokens = integer("output_tokens").default(0) + val totalTokens = integer("total_tokens").default(0) + val durationMs = long("duration_ms").default(0) + val isError = integer("is_error").default(0) // 0 = success, 1 = error + + override val primaryKey = PrimaryKey(id) +} + +/** + * Aggregated LLM usage stats for a single instance+model combination within a time range. + * + * Returned by [LlmUsageRepository.queryGroupedByInstance] — one row per unique + * `(COALESCE(instance_id, provider), model)` pair. + * + * @param instanceKey The grouping key — `instanceId` when present, otherwise `provider`. + * @param provider Raw provider identifier (e.g. "openai"). + * @param model Model name as reported by the provider. + * @param calls Total number of calls in the period (success + error). + * @param tokens Sum of [LlmUsageRecord.totalTokens] for all calls. + * @param avgDurationMs Average wall-clock duration across all calls (0 if no calls). + * @param errors Number of calls where [LlmUsageRecord.isError] = true. + */ +data class LlmInstanceStats( + val instanceKey: String, + val provider: String, + val model: String, + val calls: Int, + val tokens: Long, + val avgDurationMs: Long, + val errors: Int, +) diff --git a/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRepository.kt b/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRepository.kt new file mode 100644 index 000000000..174bea6e0 --- /dev/null +++ b/shared/src/main/kotlin/io/askimo/core/telemetry/LlmUsageRepository.kt @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: AGPLv3 + * + * Copyright (c) 2026 Askimo + */ +package io.askimo.core.telemetry + +import io.askimo.core.db.AbstractSQLiteRepository +import io.askimo.core.db.DatabaseManager +import io.askimo.core.db.SQLiteInstantColumnType +import io.askimo.core.logging.logger +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import java.time.Instant + +/** + * Repository for persisting and querying individual [LlmUsageRecord] rows. + * + * All timestamp comparisons are performed in UTC. The [LlmUsageRecordTable.timestamp] + * column is indexed to keep range-query latency low even as the table grows. + * + * Range queries (anything involving "FROM … TO …") use JDBC string comparison directly, + * because the [io.askimo.core.db.SQLiteInstantColumnType] stores instants as ISO-8601 UTC + * text which is lexicographically ordered — string `>=` / `<` gives correct temporal order. + */ +class LlmUsageRepository internal constructor( + databaseManager: DatabaseManager = DatabaseManager.getInstance(), +) : AbstractSQLiteRepository(databaseManager) { + + private val log = logger() + + fun insert(record: LlmUsageRecord) { + transaction(database) { + LlmUsageRecordTable.insert { + it[timestamp] = record.timestamp + it[provider] = record.provider + it[model] = record.model + it[instanceId] = record.instanceId + it[promptTokens] = record.promptTokens + it[outputTokens] = record.outputTokens + it[totalTokens] = record.totalTokens + it[durationMs] = record.durationMs + it[isError] = if (record.isError) 1 else 0 + } + } + log.debug( + "Inserted LLM usage record: provider={}, model={}, tokens={}, error={}", + record.provider, + record.model, + record.totalTokens, + record.isError, + ) + } + + /** + * Counts the number of calls (including errors) within [[from], [to]). + */ + fun countByPeriod(from: Instant, to: Instant): Int { + val fromStr = fmt(from) + val toStr = fmt(to) + dataSource.connection.use { conn -> + conn.prepareStatement( + "SELECT COUNT(*) FROM llm_usage_records " + + "WHERE timestamp >= ? AND timestamp < ?", + ).use { stmt -> + stmt.setString(1, fromStr) + stmt.setString(2, toStr) + stmt.executeQuery().use { rs -> + if (rs.next()) return rs.getInt(1) + } + } + } + return 0 + } + + /** + * Returns per-instance aggregated stats within [[from], [to]). + * + * Groups by `COALESCE(instance_id, provider), model` and orders by total tokens descending, + * so the highest-usage model appears first. One [LlmInstanceStats] row per unique combination. + * + * Uses JDBC directly for the same reason as other range queries — ISO-8601 string + * comparison is correct and avoids Exposed custom-column-type limitations. + */ + fun queryGroupedByInstance(from: Instant, to: Instant): List { + val fromStr = fmt(from) + val toStr = fmt(to) + val result = mutableListOf() + dataSource.connection.use { conn -> + conn.prepareStatement( + "SELECT " + + " COALESCE(instance_id, provider) AS instance_key, " + + " provider, " + + " model, " + + " COUNT(*) AS calls, " + + " COALESCE(SUM(total_tokens), 0) AS tokens, " + + " COALESCE(AVG(duration_ms), 0) AS avg_duration_ms, " + + " SUM(CASE WHEN is_error = 1 THEN 1 ELSE 0 END) AS errors " + + "FROM llm_usage_records " + + "WHERE timestamp >= ? AND timestamp < ? " + + "GROUP BY COALESCE(instance_id, provider), model " + + "ORDER BY tokens DESC", + ).use { stmt -> + stmt.setString(1, fromStr) + stmt.setString(2, toStr) + stmt.executeQuery().use { rs -> + while (rs.next()) { + result += LlmInstanceStats( + instanceKey = rs.getString("instance_key"), + provider = rs.getString("provider"), + model = rs.getString("model"), + calls = rs.getInt("calls"), + tokens = rs.getLong("tokens"), + avgDurationMs = rs.getLong("avg_duration_ms"), + errors = rs.getInt("errors"), + ) + } + } + } + } + return result + } + + /** Format an [Instant] to the ISO-8601 UTC string used by [io.askimo.core.db.SQLiteInstantColumnType]. */ + private fun fmt(instant: Instant): String = SQLiteInstantColumnType.FORMATTER.format(instant) + + /** Parse an ISO-8601 UTC string back to [Instant] (tolerant, delegates to [SQLiteInstantColumnType]). */ + private fun parseInstant(raw: String): Instant = Instant.parse(if (raw.endsWith('Z') || raw.contains('+')) raw else "${raw}Z") + + private fun ResultRow.toRecord() = LlmUsageRecord( + id = this[LlmUsageRecordTable.id], + timestamp = this[LlmUsageRecordTable.timestamp], + provider = this[LlmUsageRecordTable.provider], + model = this[LlmUsageRecordTable.model], + instanceId = this[LlmUsageRecordTable.instanceId], + promptTokens = this[LlmUsageRecordTable.promptTokens], + outputTokens = this[LlmUsageRecordTable.outputTokens], + totalTokens = this[LlmUsageRecordTable.totalTokens], + durationMs = this[LlmUsageRecordTable.durationMs], + isError = this[LlmUsageRecordTable.isError] != 0, + ) +} diff --git a/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryCollector.kt b/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryCollector.kt index 0ae857a10..8c1458bdf 100644 --- a/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryCollector.kt +++ b/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryCollector.kt @@ -9,115 +9,44 @@ import io.askimo.core.logging.logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.serialization.Serializable -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong +import java.time.Instant /** - * Collects telemetry data for RAG operations and LLM calls. - * All data stays local - no external reporting. + * Tracks LLM usage across the application lifetime. * - * Thread-safe for concurrent access. + * Every AI call — successful or failed — is recorded with its provider, model, token counts, + * and latency. This data drives the token-usage dashboard in the Discover view and the + * diagnostics export, giving users visibility into how they consume their AI quota. + * + * A **session** represents a continuous period of usage. Calling [reset] starts a new session, + * narrowing the dashboard to calls made after that point. All historical data is always retained + * and can be queried at any time through [usageRepository]. + * + * UI components observe [refreshSignal] to know when new data is available and should re-query + * the repository. */ -class TelemetryCollector { +class TelemetryCollector( + val usageRepository: LlmUsageRepository, +) { private val log = logger() - // Reactive state for UI - private val _metricsFlow = MutableStateFlow(TelemetryMetrics.empty()) - val metricsFlow: StateFlow = _metricsFlow.asStateFlow() + @Volatile + private var _sessionStart: Instant = Instant.EPOCH - // RAG Classification Metrics - private val ragClassificationTotal = AtomicInteger(0) - private val ragTriggered = AtomicInteger(0) - private val ragSkipped = AtomicInteger(0) - private val ragClassificationTotalTime = AtomicLong(0L) + /** Start of the current session window (updated by [reset]). */ + val sessionStart: Instant get() = _sessionStart - // RAG Retrieval Metrics - private val ragRetrievalTotal = AtomicInteger(0) - private val ragRetrievalTotalTime = AtomicLong(0L) - private val ragChunksRetrievedTotal = AtomicInteger(0) - - // LLM Call Metrics — keyed by "$instanceId:$model" or "$provider:$model" (legacy fallback) - private val llmCallsByInstance = ConcurrentHashMap() - private val llmTokensByInstance = ConcurrentHashMap() - private val llmDurationByInstance = ConcurrentHashMap() - private val llmErrorsByInstance = ConcurrentHashMap() - - init { - // Load persisted telemetry data on initialization - loadPersistedData() - } + private val _refreshSignal = MutableStateFlow(0L) /** - * Loads persisted telemetry data and restores counters. + * Increments each time a new [LlmUsageRecord] is written. + * Collect this in UI composables and re-query [usageRepository] on each emission. */ - private fun loadPersistedData() { - val loaded = TelemetryPersistenceManager.load() - - // Restore RAG metrics - ragClassificationTotal.set(loaded.ragClassificationTotal) - ragTriggered.set(loaded.ragTriggered) - ragSkipped.set(loaded.ragSkipped) - ragClassificationTotalTime.set(loaded.ragAvgClassificationTimeMs * loaded.ragClassificationTotal) - ragRetrievalTotal.set(loaded.ragRetrievalTotal) - ragRetrievalTotalTime.set(loaded.ragAvgRetrievalTimeMs * loaded.ragRetrievalTotal) - ragChunksRetrievedTotal.set((loaded.ragAvgChunksRetrieved * loaded.ragRetrievalTotal).toInt()) - - // Restore LLM metrics - loaded.llmCallsByInstance.forEach { (key, calls) -> - llmCallsByInstance[key] = AtomicInteger(calls) - } - loaded.llmTokensByInstance.forEach { (key, tokens) -> - llmTokensByInstance[key] = AtomicLong(tokens) - } - loaded.llmAvgDurationMsByInstance.forEach { (key, avgDuration) -> - val calls = loaded.llmCallsByInstance[key] ?: 0 - if (calls > 0) llmDurationByInstance[key] = AtomicLong(avgDuration * calls) - } - loaded.llmErrorsByInstance.forEach { (key, errors) -> - llmErrorsByInstance[key] = AtomicInteger(errors) - } - - updateMetricsFlow() - - if (loaded != TelemetryMetrics.empty()) { - log.debug("Restored telemetry: ${loaded.ragClassificationTotal} classifications, ${loaded.totalTokensUsed} tokens") - } - } + val refreshSignal: StateFlow = _refreshSignal.asStateFlow() /** - * Records a RAG classification decision. - */ - fun recordRAGClassification(triggered: Boolean, durationMs: Long) { - ragClassificationTotal.incrementAndGet() - ragClassificationTotalTime.addAndGet(durationMs) - - if (triggered) { - ragTriggered.incrementAndGet() - log.debug("RAG triggered (total: ${ragTriggered.get()}/${ragClassificationTotal.get()})") - } else { - ragSkipped.incrementAndGet() - log.debug("RAG skipped (total: ${ragSkipped.get()}/${ragClassificationTotal.get()})") - } - } - - /** - * Records a RAG retrieval operation. - */ - fun recordRAGRetrieval(chunksRetrieved: Int, durationMs: Long) { - ragRetrievalTotal.incrementAndGet() - ragRetrievalTotalTime.addAndGet(durationMs) - ragChunksRetrievedTotal.addAndGet(chunksRetrieved) - - log.debug("RAG retrieval: $chunksRetrieved chunks in ${durationMs}ms") - } - - /** - * Records an LLM call (from LangChain4J listener). - * - * The aggregation key is `"$instanceId:$model"` when [instanceId] is supplied, - * or `"$provider:$model"` as a legacy fallback when no instance is known. + * Records a successful LLM call. + * Persists a [LlmUsageRecord] row and bumps [refreshSignal]. */ fun recordLLMCall( provider: String, @@ -126,22 +55,26 @@ class TelemetryCollector { durationMs: Long, instanceId: String? = null, ) { + usageRepository.insert( + LlmUsageRecord( + provider = provider, + model = model, + instanceId = instanceId, + promptTokens = tokenUsage?.inputTokenCount() ?: 0, + outputTokens = tokenUsage?.outputTokenCount() ?: 0, + totalTokens = tokenUsage?.totalTokenCount() ?: 0, + durationMs = durationMs, + isError = false, + ), + ) val key = "${instanceId ?: provider}:$model" - - llmCallsByInstance.getOrPut(key) { AtomicInteger(0) }.incrementAndGet() - llmDurationByInstance.getOrPut(key) { AtomicLong(0L) }.addAndGet(durationMs) - tokenUsage?.let { - llmTokensByInstance.getOrPut(key) { AtomicLong(0L) }.addAndGet(it.totalTokenCount().toLong()) - } - log.debug("LLM call to $key: ${tokenUsage?.totalTokenCount() ?: 0} tokens in ${durationMs}ms") - updateMetricsFlow() + _refreshSignal.value++ } /** * Records an LLM error. - * - * Uses the same key scheme as [recordLLMCall]. + * Persists a [LlmUsageRecord] row with [LlmUsageRecord.isError] = true and bumps [refreshSignal]. */ fun recordLLMError( provider: String, @@ -149,112 +82,27 @@ class TelemetryCollector { error: Throwable, instanceId: String? = null, ) { + usageRepository.insert( + LlmUsageRecord( + provider = provider, + model = model, + instanceId = instanceId, + isError = true, + ), + ) val key = "${instanceId ?: provider}:$model" - llmErrorsByInstance.getOrPut(key) { AtomicInteger(0) }.incrementAndGet() log.warn("LLM error for $key: ${error.message}") + _refreshSignal.value++ } /** - * Gets current metrics snapshot. - */ - fun getMetrics(): TelemetryMetrics { - val classificationCount = ragClassificationTotal.get() - val retrievalCount = ragRetrievalTotal.get() - - return TelemetryMetrics( - // RAG Classification - ragClassificationTotal = classificationCount, - ragTriggered = ragTriggered.get(), - ragSkipped = ragSkipped.get(), - ragTriggeredPercent = if (classificationCount > 0) ragTriggered.get() * 100.0 / classificationCount else 0.0, - ragAvgClassificationTimeMs = if (classificationCount > 0) ragClassificationTotalTime.get() / classificationCount else 0L, - // RAG Retrieval - ragRetrievalTotal = retrievalCount, - ragAvgRetrievalTimeMs = if (retrievalCount > 0) ragRetrievalTotalTime.get() / retrievalCount else 0L, - ragAvgChunksRetrieved = if (retrievalCount > 0) ragChunksRetrievedTotal.get().toDouble() / retrievalCount else 0.0, - // LLM Calls - llmCallsByInstance = llmCallsByInstance.mapValues { it.value.get() }, - llmTokensByInstance = llmTokensByInstance.mapValues { it.value.get() }, - llmAvgDurationMsByInstance = llmDurationByInstance.mapValues { (key, totalDuration) -> - val calls = llmCallsByInstance[key]?.get() ?: 0 - if (calls > 0) totalDuration.get() / calls else 0L - }, - llmErrorsByInstance = llmErrorsByInstance.mapValues { it.value.get() }, - ) - } - - /** - * Updates the reactive state flow with current metrics and persists to disk. - */ - private fun updateMetricsFlow() { - val currentMetrics = getMetrics() - _metricsFlow.value = currentMetrics - - // Persist to disk (async, non-blocking) - TelemetryPersistenceManager.save(currentMetrics) - } - - /** - * Resets all metrics (useful for testing or per-session tracking). + * Advances [sessionStart] to now, resetting the UI view to an empty session. + * SQLite records are intentionally **kept** — query [usageRepository] directly for + * historical data. [refreshSignal] is reset to 0. */ fun reset() { - ragClassificationTotal.set(0) - ragTriggered.set(0) - ragSkipped.set(0) - ragClassificationTotalTime.set(0L) - ragRetrievalTotal.set(0) - ragRetrievalTotalTime.set(0L) - ragChunksRetrievedTotal.set(0) - llmCallsByInstance.clear() - llmTokensByInstance.clear() - llmDurationByInstance.clear() - llmErrorsByInstance.clear() - log.info("Telemetry metrics reset") - updateMetricsFlow() - TelemetryPersistenceManager.delete() - } -} - -/** - * Snapshot of telemetry metrics at a point in time. - */ -@Serializable -data class TelemetryMetrics( - // RAG Classification - val ragClassificationTotal: Int, - val ragTriggered: Int, - val ragSkipped: Int, - val ragTriggeredPercent: Double, - val ragAvgClassificationTimeMs: Long, - - // RAG Retrieval - val ragRetrievalTotal: Int, - val ragAvgRetrievalTimeMs: Long, - val ragAvgChunksRetrieved: Double, - - // LLM Calls — key is "$instanceId:$model" or "$provider:$model" (legacy) - val llmCallsByInstance: Map = emptyMap(), - val llmTokensByInstance: Map = emptyMap(), - val llmAvgDurationMsByInstance: Map = emptyMap(), - val llmErrorsByInstance: Map = emptyMap(), -) { - val totalTokensUsed: Long - get() = llmTokensByInstance.values.sum() - - companion object { - fun empty() = TelemetryMetrics( - ragClassificationTotal = 0, - ragTriggered = 0, - ragSkipped = 0, - ragTriggeredPercent = 0.0, - ragAvgClassificationTimeMs = 0L, - ragRetrievalTotal = 0, - ragAvgRetrievalTimeMs = 0L, - ragAvgChunksRetrieved = 0.0, - llmCallsByInstance = emptyMap(), - llmTokensByInstance = emptyMap(), - llmAvgDurationMsByInstance = emptyMap(), - llmErrorsByInstance = emptyMap(), - ) + _sessionStart = Instant.now() + _refreshSignal.value = 0L + log.info("Telemetry session reset — new sessionStart=$_sessionStart") } } diff --git a/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryPersistenceManager.kt b/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryPersistenceManager.kt deleted file mode 100644 index dcba45635..000000000 --- a/shared/src/main/kotlin/io/askimo/core/telemetry/TelemetryPersistenceManager.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: AGPLv3 - * - * Copyright (c) 2026 Askimo - */ -package io.askimo.core.telemetry - -import io.askimo.core.logging.logger -import io.askimo.core.util.AskimoHome -import io.askimo.core.util.appJson -import kotlinx.serialization.SerializationException -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardOpenOption - -/** - * Manages persistence of telemetry data to disk. - * Saves and loads telemetry metrics to/from a JSON file in the user's home directory. - */ -object TelemetryPersistenceManager { - private val log = logger() - - /** Path to the telemetry data file */ - private val telemetryPath: Path = AskimoHome.base().resolve("telemetry.json") - - /** - * Saves telemetry metrics to disk. - * - * @param metrics The metrics to save - * @return true if save was successful, false otherwise - */ - fun save(metrics: TelemetryMetrics): Boolean = try { - Files.createDirectories(telemetryPath.parent) - - val json = appJson.encodeToString(TelemetryMetrics.serializer(), metrics) - Files.writeString( - telemetryPath, - json, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - ) - - log.debug("Telemetry saved to $telemetryPath") - true - } catch (e: Exception) { - log.warn("Failed to save telemetry to $telemetryPath: ${e.message}", e) - false - } - - /** - * Loads telemetry metrics from disk. - * - * @return Loaded metrics, or empty metrics if file doesn't exist or fails to load - */ - fun load(): TelemetryMetrics { - try { - if (!Files.exists(telemetryPath)) { - log.debug("Telemetry file not found at $telemetryPath. Starting with empty metrics.") - return TelemetryMetrics.empty() - } - - val json = Files.readString(telemetryPath) - val loaded = appJson.decodeFromString(json) - - log.debug("Telemetry loaded from $telemetryPath: ${loaded.ragClassificationTotal} classifications, ${loaded.totalTokensUsed} tokens") - return loaded - } catch (e: SerializationException) { - log.warn("Failed to parse telemetry file at $telemetryPath. Using empty metrics.", e) - return TelemetryMetrics.empty() - } catch (e: Exception) { - log.warn("Failed to load telemetry from $telemetryPath: ${e.message}", e) - return TelemetryMetrics.empty() - } - } - - /** - * Deletes the telemetry file from disk. - * - * @return true if deletion was successful, false otherwise - */ - fun delete(): Boolean = try { - if (Files.exists(telemetryPath)) { - Files.delete(telemetryPath) - log.debug("Telemetry file deleted: $telemetryPath") - true - } else { - log.debug("Telemetry file does not exist: $telemetryPath") - false - } - } catch (e: Exception) { - log.warn("Failed to delete telemetry file at $telemetryPath: ${e.message}", e) - false - } -}