From c6147ce4050361e8454c96900fb94ff98b4675c9 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 10 Dec 2025 14:00:52 +0200 Subject: [PATCH] feat: [PowerSync] Add metadata tracking --- .changeset/some-wombats-rest.md | 39 +++++++ docs/collections/powersync-collection.md | 103 +++++++++++++++++ .../src/PowerSyncTransactor.ts | 106 ++++++++++++++---- .../src/definitions.ts | 5 + .../powersync-db-collection/src/powersync.ts | 3 +- .../tests/powersync.test.ts | 70 +++++++++++- 6 files changed, 299 insertions(+), 27 deletions(-) create mode 100644 .changeset/some-wombats-rest.md diff --git a/.changeset/some-wombats-rest.md b/.changeset/some-wombats-rest.md new file mode 100644 index 000000000..649af6fb1 --- /dev/null +++ b/.changeset/some-wombats-rest.md @@ -0,0 +1,39 @@ +--- +"@tanstack/powersync-db-collection": patch +--- + +Added support for tracking collection operation metadata in PowerSync CrudEntry operations. + +```typescript +// Schema config +const APP_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + author: column.text, + created_at: column.text, + }, + { + // Metadata tracking must be enabled on the PowerSync table + trackMetadata: true, + } + ), +}) + +// ... Other config + +// Collection operations which specify metadata +await collection.insert( + { + id, + name: `document`, + author: `Foo`, + }, + // The string version of this will be present in PowerSync `CrudEntry`s during uploads + { + metadata: { + extraInfo: "Info", + }, + } +) +``` diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index afc665836..36c2d8be2 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -401,6 +401,109 @@ task.due_date.getTime() // OK - TypeScript knows this is a Date Updates to the collection are applied optimistically to the local state first, then synchronized with PowerSync and the backend. If an error occurs during sync, the changes are automatically rolled back. +### Metadata Tracking + +Metadata tracking allows attaching custom metadata to collection operations (insert, update, delete). This metadata is persisted alongside the operation and available in PowerSync `CrudEntry` records during upload processing. This is useful for passing additional context about mutations to the backend, such as audit information, operation sources, or custom processing hints. + +#### Enabling Metadata Tracking + +Metadata tracking must be enabled on the PowerSync table: + +```typescript +const APP_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + author: column.text, + }, + { + // Enable metadata tracking on this table + trackMetadata: true, + } + ), +}) +``` + +#### Using Metadata in Operations + +Once enabled, metadata can be passed to any collection operation: + +```typescript +const documents = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.documents, + }) +) + +// Insert with metadata +await documents.insert( + { + id: crypto.randomUUID(), + name: "Report Q4", + author: "Jane Smith", + }, + { + metadata: { + source: "web-app", + userId: "user-123", + timestamp: Date.now(), + }, + } +).isPersisted.promise + +// Update with metadata +await documents.update( + docId, + { metadata: { reason: "typo-fix", editor: "user-456" } }, + (doc) => { + doc.name = "Report Q4 (Updated)" + } +).isPersisted.promise + +// Delete with metadata +await documents.delete(docId, { + metadata: { deletedBy: "user-789", reason: "duplicate" }, +}).isPersisted.promise +``` + +#### Accessing Metadata During Upload + +The metadata is available in PowerSync `CrudEntry` records when processing uploads in the connector: + +```typescript +import { CrudEntry } from "@powersync/web" + +class Connector implements PowerSyncBackendConnector { + // ... + + async uploadData(database: AbstractPowerSyncDatabase) { + const batch = await database.getCrudBatch() + if (!batch) return + + for (const entry of batch.crud) { + console.log("Operation:", entry.op) // PUT, PATCH, DELETE + console.log("Table:", entry.table) + console.log("Data:", entry.opData) + console.log("Metadata:", entry.metadata) // Custom metadata (stringified) + + // Parse metadata if needed + if (entry.metadata) { + const meta = JSON.parse(entry.metadata) + console.log("Source:", meta.source) + console.log("User ID:", meta.userId) + } + + // Process the operation with the backend... + } + + await batch.complete() + } +} +``` + +**Note**: If metadata is provided to an operation but the table doesn't have `trackMetadata: true`, a warning will be logged and the metadata will be ignored. + ## Configuration Options The `powerSyncCollectionOptions` function accepts the following options: diff --git a/packages/powersync-db-collection/src/PowerSyncTransactor.ts b/packages/powersync-db-collection/src/PowerSyncTransactor.ts index ceed7ed20..81520f02c 100644 --- a/packages/powersync-db-collection/src/PowerSyncTransactor.ts +++ b/packages/powersync-db-collection/src/PowerSyncTransactor.ts @@ -1,11 +1,14 @@ import { sanitizeSQL } from "@powersync/common" import DebugModule from "debug" -import { asPowerSyncRecord, mapOperationToPowerSync } from "./helpers" import { PendingOperationStore } from "./PendingOperationStore" +import { asPowerSyncRecord, mapOperationToPowerSync } from "./helpers" import type { AbstractPowerSyncDatabase, LockContext } from "@powersync/common" import type { PendingMutation, Transaction } from "@tanstack/db" -import type { EnhancedPowerSyncCollectionConfig } from "./definitions" import type { PendingOperation } from "./PendingOperationStore" +import type { + EnhancedPowerSyncCollectionConfig, + PowerSyncCollectionMeta, +} from "./definitions" const debug = DebugModule.debug(`ts/db:powersync`) @@ -160,6 +163,13 @@ export class PowerSyncTransactor { async (tableName, mutation, serializeValue) => { const values = serializeValue(mutation.modified) const keys = Object.keys(values).map((key) => sanitizeSQL`${key}`) + const queryParameters = Object.values(values) + + const metadataValue = this.processMutationMetadata(mutation) + if (metadataValue != null) { + keys.push(`_metadata`) + queryParameters.push(metadataValue) + } await context.execute( ` @@ -168,7 +178,7 @@ export class PowerSyncTransactor { VALUES (${keys.map((_) => `?`).join(`, `)}) `, - Object.values(values) + queryParameters ) } ) @@ -188,6 +198,13 @@ export class PowerSyncTransactor { async (tableName, mutation, serializeValue) => { const values = serializeValue(mutation.modified) const keys = Object.keys(values).map((key) => sanitizeSQL`${key}`) + const queryParameters = Object.values(values) + + const metadataValue = this.processMutationMetadata(mutation) + if (metadataValue != null) { + keys.push(`_metadata`) + queryParameters.push(metadataValue) + } await context.execute( ` @@ -195,7 +212,7 @@ export class PowerSyncTransactor { SET ${keys.map((key) => `${key} = ?`).join(`, `)} WHERE id = ? `, - [...Object.values(values), asPowerSyncRecord(mutation.modified).id] + [...queryParameters, asPowerSyncRecord(mutation.modified).id] ) } ) @@ -213,12 +230,26 @@ export class PowerSyncTransactor { context, waitForCompletion, async (tableName, mutation) => { - await context.execute( - ` - DELETE FROM ${tableName} WHERE id = ? - `, - [asPowerSyncRecord(mutation.original).id] - ) + const metadataValue = this.processMutationMetadata(mutation) + if (metadataValue != null) { + /** + * Delete operations with metadata require a different approach to handle metadata. + * This will delete the record. + */ + await context.execute( + ` + UPDATE ${tableName} SET _deleted = TRUE, _metadata = ? WHERE id = ? + `, + [metadataValue, asPowerSyncRecord(mutation.original).id] + ) + } else { + await context.execute( + ` + DELETE FROM ${tableName} WHERE id = ? + `, + [asPowerSyncRecord(mutation.original).id] + ) + } } ) } @@ -239,17 +270,8 @@ export class PowerSyncTransactor { serializeValue: (value: any) => Record ) => Promise ): Promise { - if ( - typeof (mutation.collection.config as any).utils?.getMeta != `function` - ) { - throw new Error(`Could not get tableName from mutation's collection config. - The provided mutation might not have originated from PowerSync.`) - } - - const { tableName, trackedTableName, serializeValue } = ( - mutation.collection - .config as unknown as EnhancedPowerSyncCollectionConfig - ).utils.getMeta() + const { tableName, trackedTableName, serializeValue } = + this.getMutationCollectionMeta(mutation) await handler(sanitizeSQL`${tableName}`, mutation, serializeValue) @@ -268,4 +290,46 @@ export class PowerSyncTransactor { timestamp: diffOperation.timestamp, } } + + protected getMutationCollectionMeta( + mutation: PendingMutation + ): PowerSyncCollectionMeta { + if ( + typeof (mutation.collection.config as any).utils?.getMeta != `function` + ) { + throw new Error(`Collection is not a PowerSync collection.`) + } + return ( + mutation.collection + .config as unknown as EnhancedPowerSyncCollectionConfig + ).utils.getMeta() + } + + /** + * Processes collection mutation metadata for persistence to the database. + * We only support storing string metadata. + * @returns null if no metadata should be stored. + */ + protected processMutationMetadata( + mutation: PendingMutation + ): string | null { + const { metadataIsTracked } = this.getMutationCollectionMeta(mutation) + if (!metadataIsTracked) { + // If it's not supported, we don't store metadata. + if (typeof mutation.metadata != `undefined`) { + // Log a warning if metadata is provided but not tracked. + this.database.logger.warn( + `Metadata provided for collection ${mutation.collection.id} but the PowerSync table does not track metadata. The PowerSync table should be configured with trackMetadata: true.`, + mutation.metadata + ) + } + return null + } else if (typeof mutation.metadata == `undefined`) { + return null + } else if (typeof mutation.metadata == `string`) { + return mutation.metadata + } else { + return JSON.stringify(mutation.metadata) + } + } } diff --git a/packages/powersync-db-collection/src/definitions.ts b/packages/powersync-db-collection/src/definitions.ts index d27b47311..8dd2433c6 100644 --- a/packages/powersync-db-collection/src/definitions.ts +++ b/packages/powersync-db-collection/src/definitions.ts @@ -246,6 +246,11 @@ export type PowerSyncCollectionMeta = { * Serializes a collection value to the SQLite type */ serializeValue: (value: any) => ExtractedTable + + /** + * Whether the PowerSync table tracks metadata. + */ + metadataIsTracked: boolean } /** diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 549b08db0..9996c8143 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -242,7 +242,7 @@ export function powerSyncCollectionOptions< // The collection output type type OutputType = InferPowerSyncOutputType - const { viewName } = table + const { viewName, trackMetadata: metadataIsTracked } = table /** * Deserializes data from the incoming sync stream @@ -459,6 +459,7 @@ export function powerSyncCollectionOptions< getMeta: () => ({ tableName: viewName, trackedTableName, + metadataIsTracked, serializeValue: (value) => serializeForSQLite( value, diff --git a/packages/powersync-db-collection/tests/powersync.test.ts b/packages/powersync-db-collection/tests/powersync.test.ts index 5d71af023..2e7a80ece 100644 --- a/packages/powersync-db-collection/tests/powersync.test.ts +++ b/packages/powersync-db-collection/tests/powersync.test.ts @@ -23,11 +23,16 @@ const APP_SCHEMA = new Schema({ name: column.text, active: column.integer, // Will be mapped to Boolean }), - documents: new Table({ - name: column.text, - author: column.text, - created_at: column.text, // Will be mapped to Date - }), + documents: new Table( + { + name: column.text, + author: column.text, + created_at: column.text, // Will be mapped to Date + }, + { + trackMetadata: true, + } + ), }) describe(`PowerSync Integration`, () => { @@ -325,6 +330,60 @@ describe(`PowerSync Integration`, () => { .every((crudEntry) => crudEntry.transactionId == lastTransactionId) ).true }) + + /** + * Metadata provided by the collection operation should be persisted to the database if supported by the SQLite table. + */ + it(`should persist collection operation metadata`, async () => { + const db = await createDatabase() + + const collection = createDocumentsCollection(db) + await collection.stateWhenReady() + + const metadata = { + text: `some text`, + number: 123, + boolean: true, + } + const id = randomUUID() + await collection.insert( + { + id, + name: `new`, + author: `somebody`, + }, + { + metadata, + } + ).isPersisted.promise + + // Now do an update + await collection.update( + id, + { metadata: metadata }, + (d) => (d.name = `updatedNew`) + ).isPersisted.promise + + await collection.delete(id, { metadata }).isPersisted.promise + + // There should be a crud entries for this + const _crudEntries = await db.getAll(` + SELECT * FROM ps_crud ORDER BY id`) + + const crudEntries = _crudEntries.map((r) => CrudEntry.fromRow(r)) + + // The metadata should be available in the CRUD entries for upload + const stringifiedMetadata = JSON.stringify(metadata) + expect(crudEntries.length).toBe(3) + expect(crudEntries[0]!.metadata).toEqual(stringifiedMetadata) + expect(crudEntries[1]!.metadata).toEqual(stringifiedMetadata) + expect(crudEntries[2]!.metadata).toEqual(stringifiedMetadata) + + // Verify the item is deleted from SQLite + const documents = await db.getAll(` + SELECT * FROM documents`) + expect(documents.length).toBe(0) + }) }) describe(`General use`, () => { @@ -340,6 +399,7 @@ describe(`PowerSync Integration`, () => { vi.spyOn(options.utils, `getMeta`).mockImplementation(() => ({ tableName: `fakeTable`, trackedTableName: `error`, + metadataIsTracked: true, serializeValue: () => ({}) as any, })) // Create two collections for the same table