FS-DB wip

apps/web/content/articles/journey-fixing-the-data-layer.mdx

@@ -0,0 +1,16 @@
+---
+meta_title: "Messy journey of fixing the data layer of Hyprnote"
+meta_description: "TOOD"
+author: "Yujong Lee"
+coverImage: "/api/images/blog/journey-fixing-the-data-layer/cover.png"
+published: true
+date: "2026-01-25"
+---
+
+## How did we get here?
+
+## The problems
+
+## The solutions
+
+## The results

plugins/fs-db/AGENTS.md

@@ -0,0 +1,5 @@
+We are working on designing `plugins/fs-db/src`. Still very WIP. 
+
+See `plugins/fs-db/SPEC.md` and `plugins/fs-db/DECISIONS.md`.
+
+Your job is have conversation with me, incrementally form a better spec, document decisions.

plugins/fs-db/Cargo.toml

@@ -12,6 +12,7 @@ tauri-plugin = { workspace = true, features = ["build"] }
 
 [dev-dependencies]
 specta-typescript = { workspace = true }
+tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }
 
 [dependencies]
@@ -20,7 +21,9 @@ tauri-plugin-settings = { workspace = true }
 tauri-specta = { workspace = true, features = ["derive", "typescript"] }
 
 serde = { workspace = true }
+serde_json = { workspace = true }
 specta = { workspace = true }
 
+semver = "1"
 thiserror = { workspace = true }
 uuid = { workspace = true }

plugins/fs-db/DECISIONS.md

@@ -0,0 +1,113 @@
+---
+Note that DECISIONS file is not for writing open questions. This is only for documenting the decisions we agreed.
+---
+
+## Migration Model
+
+**Decision**: Operation-based migrations
+
+Each migration is a function that receives `base_dir: &Path` and performs whatever operations needed:
+
+```rust
+pub struct Migration {
+    pub name: &'static str,
+    pub from: SchemaVersion,
+    pub to: SchemaVersion,
+    pub run: fn(&Path) -> Result<()>,
+}
+```
+
+**Rationale**:
+- Migrations are diverse (SQLite extraction vs file rename vs frontmatter transform)
+- Simple and flexible - no artificial constraints
+- Can add structure later if patterns emerge
+
+## Migration Ordering
+
+**Decision**: Use `semver::Version` for app version-based migrations
+
+Use the `semver` crate. Migrations are tied to actual Hyprnote app versions, not arbitrary schema numbers. Not every app version has a migration - only versions that need data structure changes (checkpoints).
+
+```rust
+// https://docs.rs/semver/latest/semver/
+use semver::Version;
+
+Migration::new(
+    "extract_from_sqlite",
+    Version::parse("1.0.1").unwrap(),
+    Version::parse("1.0.2").unwrap(),
+    |base_dir| { ... }
+)
+```
+
+## App Version Source
+
+**Decision**: Get current app version from tauri's `Config.version`
+
+See https://docs.rs/tauri/latest/tauri/struct.Config.html#structfield.version
+
+## Fresh Install vs Migration Decision Logic
+
+```
+if .schema/version exists:
+    current_version = parse(.schema/version)
+    run migrations where from >= current_version
+else if db.sqlite exists:
+    # Old user before version tracking
+    current_version = "1.0.1"  # or earliest known version
+    run migrations where from >= current_version
+else:
+    # Fresh install, no data to migrate
+    write app_version to .schema/version
+    skip all migrations
+```
+
+## Runner Algorithm
+
+1. Determine `current_version` using decision logic above
+2. Get `app_version` from tauri Config (https://docs.rs/tauri/latest/tauri/struct.Config.html#structfield.version)
+3. Collect all migrations where `from >= current_version && to <= app_version`
+4. Sort by `from` using semver's `Ord`
+5. Run in order
+6. Write `app_version` to `.schema/version`
+
+## Testing
+
+Tests should make migration behavior explicit:
+
+```rust
+#[test]
+fn fresh_install_skips_all_migrations() {
+    // Given: empty base_dir (no .schema/version, no db.sqlite)
+    // When: app v1.0.3 runs
+    // Then: no migrations run, .schema/version written as "1.0.3"
+}
+
+#[test]
+fn old_user_without_version_file_but_has_sqlite() {
+    // Given: db.sqlite exists, no .schema/version
+    // When: app v1.0.3 runs
+    // Then: treat as v1.0.1, run all migrations from 1.0.1
+}
+
+#[test]
+fn user_on_1_0_1_upgrading_to_1_0_3() {
+    // Given: .schema/version = "1.0.1"
+    // When: app v1.0.3 runs
+    // Then: migrations 1.0.1→1.0.2 and 1.0.2→1.0.3 are applied
+}
+
+#[test]
+fn user_on_1_0_2_upgrading_to_1_0_3() {
+    // Given: .schema/version = "1.0.2"
+    // When: app v1.0.3 runs
+    // Then: only migration 1.0.2→1.0.3 is applied
+}
+
+#[test]
+fn user_already_on_latest() {
+    // Given: .schema/version = "1.0.3"
+    // When: app v1.0.3 runs
+    // Then: no migrations run
+}
+```

plugins/fs-db/SPEC.md

@@ -0,0 +1,29 @@
+---
+Note that SPEC file is not a implementation plan.
+Use this for requirment understanding, not planning.
+Ask followup question if needed, and update it if I explicitly agreed.
+---
+
+## Where we're heading
+
+Mostly 2 things.
+
+1. Lazy loading for some part of data.
+    - We already doing some kind of file-system based storage thing.(See apps/desktop/src/store/tinybase) (like obsidian, but bit more complex because we have more richer/relational data)
+    - Due to performance reason, we are migrating off from "loading everything to Tinybase" to "load only metadatas - for listing session purpose etc - and load detailed data on-demand.
+    - **Scope**: Session is the priority (includes transcripts, enhanced_notes). Other entities (template, chat_shortcut, etc.) are not priority for lazy loading.
+
+2. SQlite like migration support, for filesystem structure. (plugins/fs-db)
+    - Migration run in `setup` of `plugins/fs-db/src/lib.rs`. Resolve past<>current-app-version and apply logic sequencly.
+    - **Version tracking**: Global `.schema/version` file.
+    - We need test to ensure the user's data is properly migrated to latest structure, when they do OTA update.
+    - See https://github.com/fastrepl/hyprnote-data. Feel free to clone it in `/tmp/hyprnote-data` and inspect.
+    - We might bring `plugins/importer/src/sources/hyprnote/v1_sqlite` into fs-db migration. That is level of flexibility we need.
+    - **SQLite migration**: Users on old versions use `apps/desktop/src/store/tinybase/persister/local` (SQLite). Need migration path from SQLite to filesystem-first.
+
+## Migration capabilities needed
+
+Migrations need to support:
+- Filesystem-level: rename, move, delete files/folders
+- File-level: frontmatter transform, field addition/deletion (for md files)
+- Data extraction: SQLite/TinyBase JSON → filesystem structure (for v0 → v1)

plugins/fs-db/js/bindings.gen.ts

@@ -1,86 +1,90 @@
 // @ts-nocheck
-/** tauri-specta globals **/
-import {
-  Channel as TAURI_CHANNEL,
-  invoke as TAURI_INVOKE,
-} from "@tauri-apps/api/core";
-import * as TAURI_API_EVENT from "@tauri-apps/api/event";
-import { type WebviewWindow as __WebviewWindow__ } from "@tauri-apps/api/webviewWindow";
 
 // This file was generated by [tauri-specta](https://github.com/oscartbeaumont/tauri-specta). Do not edit this file manually.
 
 /** user-defined commands **/
 
+
 export const commands = {
-  async ping(payload: PingRequest): Promise<Result<PingResponse, string>> {
+async ping(payload: PingRequest) : Promise<Result<PingResponse, string>> {
     try {
-      return {
-        status: "ok",
-        data: await TAURI_INVOKE("plugin:fs-db|ping", { payload }),
-      };
-    } catch (e) {
-      if (e instanceof Error) throw e;
-      else return { status: "error", error: e as any };
-    }
-  },
-};
+    return { status: "ok", data: await TAURI_INVOKE("plugin:fs-db|ping", { payload }) };
+} catch (e) {
+    if(e instanceof Error) throw e;
+    else return { status: "error", error: e  as any };
+}
+}
+}
 
 /** user-defined events **/
 
+
+
 /** user-defined constants **/
 
+
+
 /** user-defined types **/
 
-export type PingRequest = { value: string | null };
-export type PingResponse = { value: string | null };
+export type PingRequest = { value: string | null }
+export type PingResponse = { value: string | null }
+
+/** tauri-specta globals **/
+
+import {
+	invoke as TAURI_INVOKE,
+	Channel as TAURI_CHANNEL,
+} from "@tauri-apps/api/core";
+import * as TAURI_API_EVENT from "@tauri-apps/api/event";
+import { type WebviewWindow as __WebviewWindow__ } from "@tauri-apps/api/webviewWindow";
 
 type __EventObj__<T> = {
-  listen: (
-    cb: TAURI_API_EVENT.EventCallback<T>,
-  ) => ReturnType<typeof TAURI_API_EVENT.listen<T>>;
-  once: (
-    cb: TAURI_API_EVENT.EventCallback<T>,
-  ) => ReturnType<typeof TAURI_API_EVENT.once<T>>;
-  emit: null extends T
-    ? (payload?: T) => ReturnType<typeof TAURI_API_EVENT.emit>
-    : (payload: T) => ReturnType<typeof TAURI_API_EVENT.emit>;
+	listen: (
+		cb: TAURI_API_EVENT.EventCallback<T>,
+	) => ReturnType<typeof TAURI_API_EVENT.listen<T>>;
+	once: (
+		cb: TAURI_API_EVENT.EventCallback<T>,
+	) => ReturnType<typeof TAURI_API_EVENT.once<T>>;
+	emit: null extends T
+		? (payload?: T) => ReturnType<typeof TAURI_API_EVENT.emit>
+		: (payload: T) => ReturnType<typeof TAURI_API_EVENT.emit>;
 };
 
 export type Result<T, E> =
-  | { status: "ok"; data: T }
-  | { status: "error"; error: E };
+	| { status: "ok"; data: T }
+	| { status: "error"; error: E };
 
 function __makeEvents__<T extends Record<string, any>>(
-  mappings: Record<keyof T, string>,
+	mappings: Record<keyof T, string>,
 ) {
-  return new Proxy(
-    {} as unknown as {
-      [K in keyof T]: __EventObj__<T[K]> & {
-        (handle: __WebviewWindow__): __EventObj__<T[K]>;
-      };
-    },
-    {
-      get: (_, event) => {
-        const name = mappings[event as keyof T];
-
-        return new Proxy((() => {}) as any, {
-          apply: (_, __, [window]: [__WebviewWindow__]) => ({
-            listen: (arg: any) => window.listen(name, arg),
-            once: (arg: any) => window.once(name, arg),
-            emit: (arg: any) => window.emit(name, arg),
-          }),
-          get: (_, command: keyof __EventObj__<any>) => {
-            switch (command) {
-              case "listen":
-                return (arg: any) => TAURI_API_EVENT.listen(name, arg);
-              case "once":
-                return (arg: any) => TAURI_API_EVENT.once(name, arg);
-              case "emit":
-                return (arg: any) => TAURI_API_EVENT.emit(name, arg);
-            }
-          },
-        });
-      },
-    },
-  );
+	return new Proxy(
+		{} as unknown as {
+			[K in keyof T]: __EventObj__<T[K]> & {
+				(handle: __WebviewWindow__): __EventObj__<T[K]>;
+			};
+		},
+		{
+			get: (_, event) => {
+				const name = mappings[event as keyof T];
+
+				return new Proxy((() => {}) as any, {
+					apply: (_, __, [window]: [__WebviewWindow__]) => ({
+						listen: (arg: any) => window.listen(name, arg),
+						once: (arg: any) => window.once(name, arg),
+						emit: (arg: any) => window.emit(name, arg),
+					}),
+					get: (_, command: keyof __EventObj__<any>) => {
+						switch (command) {
+							case "listen":
+								return (arg: any) => TAURI_API_EVENT.listen(name, arg);
+							case "once":
+								return (arg: any) => TAURI_API_EVENT.once(name, arg);
+							case "emit":
+								return (arg: any) => TAURI_API_EVENT.emit(name, arg);
+						}
+					},
+				});
+			},
+		},
+	);
 }

plugins/fs-db/src/error.rs

@@ -6,6 +6,12 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub enum Error {
     #[error(transparent)]
     Io(#[from] std::io::Error),
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+    #[error("version parse error: {0}")]
+    VersionParse(String),
+    #[error("migration failed")]
+    Migration,
 }
 
 impl Serialize for Error {

plugins/fs-db/src/ext.rs

@@ -1,4 +1,5 @@
 pub struct FsDb<'a, R: tauri::Runtime, M: tauri::Manager<R>> {
+    #[allow(dead_code)]
     manager: &'a M,
     _runtime: std::marker::PhantomData<fn() -> R>,
 }