diff --git a/Cargo.lock b/Cargo.lock index e7ee9cc..b163d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,12 +46,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -377,6 +421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -385,8 +430,22 @@ version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -395,6 +454,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -1097,6 +1162,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -1320,6 +1391,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" @@ -1859,6 +1936,7 @@ version = "0.1.0" dependencies = [ "async-trait", "chrono", + "clap", "criterion", "once_cell", "redis", @@ -2304,6 +2382,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2633,6 +2717,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" diff --git a/Cargo.toml b/Cargo.toml index cd58380..2212165 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,9 @@ once_cell = "1" toml = "0.8" serde_yaml = "0.9" +# CLI (ryx-rs binary) +clap = { version = "4", features = ["derive"] } + # tracing: structured, async-aware logging. We instrument every SQL execution # so users can enable RUST_LOG=ryx=debug for full query visibility. tracing = "0.1" diff --git a/README.md b/README.md index 1061dd2..2c12999 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Ryx ORM

- Django-style ORM — Python and Rust. Powered by Rust. + Django-style ORM for Python and Rust. Powered by a shared Rust SQL engine.

@@ -28,7 +28,7 @@ ```python import ryx -from ryx import Model, CharField, Q +from ryx import Model, BooleanField, CharField, IntField, Q class Post(Model): title = CharField(max_length=200) @@ -41,6 +41,7 @@ posts = await Post.objects.filter(Q(active=True) | Q(views__gte=1000)) ```rust use ryx_rs::model; +use ryx_rs::{ObjectsManager, Q}; #[model] struct Post { @@ -50,7 +51,8 @@ struct Post { active: bool, } -let posts = Post::objects() +let posts = ObjectsManager::::new() + .all() .filter(Q::or(Q::new("active", true), Q::new("views__gte", 1000))) .all().await?; ``` @@ -68,6 +70,26 @@ Full docs, guides, API reference: **[ryx.alldotpy.com](https://ryx.alldotpy.com) - [Python quick start](https://ryx.alldotpy.com/getting-started/quick-start) - [Rust quick start](https://ryx.alldotpy.com/getting-started/installation) +- [API reference](https://ryx.alldotpy.com/reference/api-reference) +- [CLI](https://ryx.alldotpy.com/advanced/cli) +- [Configuration and routing](https://ryx.alldotpy.com/advanced/configuration-and-routing) + +## Feature Map + +| Area | Python API | Rust API | +|---|---|---| +| **Models** | `Model`, `Field`, `Meta`, hooks | `#[model]`, `Model`, `FromRow`, metadata | +| **Fields** | Auto, integer, numeric, text, date/time, JSON, array, binary, relation fields | Macro-derived field metadata | +| **Queries** | Lazy async `QuerySet`, `Q`, lookups, transforms, joins, values, annotations | `QuerySet`, `Q`, lookups, values, annotations | +| **CRUD** | `create`, `save`, `get`, `first`, `count`, `update`, `delete`, `refresh_from_db` | `InsertBuilder`, `all`, `get`, `first`, `count`, `update`, `delete` | +| **Bulk/streaming** | `bulk_create`, `bulk_update`, `bulk_delete`, chunked/keyset stream | streaming chunks through `QueryStream` | +| **Transactions** | `async with transaction()` with savepoints | `transaction(|tx| async move { ... })` | +| **Migrations** | autodetect, DDL generation, runner, CLI | state diffing, DDL generation, runner | +| **Multi-db** | aliases, `.using()`, `Meta.database`, router, env/config auto-init | aliases, `.using()`, config init | +| **PostgreSQL schemas** | `.schema()`, schema-aware migrations | `.schema()` | +| **Caching** | `MemoryCache`, `QuerySet.cache()`, invalidation helpers | cache backend and cached queryset | +| **Signals/hooks** | pre/post save/delete/update and model hooks | not part of Rust surface | +| **Raw SQL** | raw fetch/execute and parameterized helpers | backend/query crates | ## Comparison @@ -76,8 +98,9 @@ Full docs, guides, API reference: **[ryx.alldotpy.com](https://ryx.alldotpy.com) | **API style** | Schema-first | Verbose builders | **Django-like** | | **Q objects (OR/AND/NOT)** | ❌ | ❌ | ✅ | | **Lookups** | Basic | Basic | **30+** | -| **select_related** | ❌ | ✅ (Eager) | ✅ | +| **select_related** | ❌ | ✅ (Eager) | Rust API; Python currently uses explicit `.join()` | | **Migrations** | Diesel CLI | sea-orm-cli | **Built-in** | +| **PostgreSQL schemas** | ❌ | ❌ | ✅ | | **Backends** | PG · MySQL · SQLite | PG · MySQL · SQLite | **PG · MySQL · SQLite** | ## Architecture diff --git a/RYK.md b/RYK.md new file mode 100644 index 0000000..b0a71c9 --- /dev/null +++ b/RYK.md @@ -0,0 +1,106 @@ +## Goal +- Complete multi-schema support for PostgreSQL across migration, query, and CLI layers in both Rust and Python, then add comprehensive tests. + +## Constraints & Preferences +- Models are **schema-agnostic** — no `#[schema]` on models. Schema is specified at query time (`.schema("name")` on QuerySet) or migration time (`.schema("name")` on runner). +- `schema=""` (default) means the backend's default schema (public for PG, ignored for MySQL/SQLite). No qualification in SQL when unset → 100% backward compat. +- Composite key `(schema, table_name)` everywhere: `diff_states`, introspection, DDL generation. +- `CREATE SCHEMA IF NOT EXISTS` is automatic in the runner (when schema is non-empty and backend supports it). +- Schema support is **PostgreSQL only** — MySQL/SQLite treat schema field as no-op (`supports_schemas()` returns `false` for non-PG). +- Performance: `Option` in QueryNode (not `Option`), `supports_schemas()` is a `const fn`, qualification done once per SQL compile (not per row). +- Rust `gen` is reserved in edition 2024 — use `ddl_gen` instead of `gen` as variable name. + +## Progress +### Done +- **Rust — Backend.supports_schemas()** added to `ryx-query/src/backend.rs` as a `const fn`. +- **Rust — TableState.schema**, **SchemaChange.schema**, **ChangeKind::CreateSchema** — all core types updated. `diff_states()` rewritten with composite key `(schema, name)`. +- **Rust — DDLGenerator.schema** with `in_schema()` builder, `qn()` helper, `create_schema()`. All DDL methods (`create_table`, `drop_table`, `add_column`, `drop_column`, `alter_column`, `create_index`, `drop_index`, `add_check_constraint`, `add_foreign_key`) use `self.qn()`. `generate()` handles `CreateSchema` in zeroth pass. +- **Rust — Operations** (`operations.rs`): `CreateSchema` variant, `schema: String` on all table operations, `schema()` accessor. +- **Rust — Autodetect** (`autodetect.rs`): `ModelEntry.schema`, `build_target()` schema apply, `changes_to_operations()` emits `CreateSchema` + schema pass-through. +- **Rust — Runner** (`runner.rs`): `FileRunner.schema`, `.schema()` builder, schema-aware `run_live()`, `plan_live()`, `apply_files()`, schema override per operation in `apply_files()`/`plan_files()`. `operation_to_sql()` handles `CreateSchema`. +- **Rust — Introspection**: PostgreSQL uses `table_schema = '{schema}'`, MySQL/SQLite set `schema: ""`. +- **Rust — Query layer** (`ryx-query/src/ast.rs`): `QueryNode.schema` (`Option`), `with_schema()` builder. Compiler (`compilr.rs`): `write_table_ref()` / `write_join_table_ref()` helpers, all compile functions (`SELECT`, `COUNT`, `DELETE`, `UPDATE`, `INSERT`, `JOIN`) use schema qualification. Plan hash includes `node.schema`. +- **Rust — QuerySet** (`ryx-rs/src/queryset.rs`): `.schema()` builder method. +- **Rust — CLI** (`ryx-rs/src/main.rs`): `--schema` flag on `Migrate` and `Sqlmigrate` subcommands, passed to `FileRunner.schema()` and `DDLGenerator.in_schema()`. +- **Python — state.py**: `TableState.schema`, `ChangeKind.CREATE_SCHEMA`, `SchemaChange.schema`, `diff_states()` composite key + CreateSchema detection. `to_json()`/`from_json()` schema-aware. +- **Python — ddl.py**: `DDLGenerator.schema` parameter, `_qn()` helper, `create_schema()`. All DDL methods use `_qn()` for table references. `generate_schema_ddl()` creates per-table schema DDLGenerator. +- **Python — autodetect.py**: `schema: str` on all operation types (`CreateTable`, `AddField`, `AlterField`, `CreateIndex`), `to_python()` serialization includes schema, `apply_migration_to_state()` preserves schema on tables, `_changes_to_operations()` passes schema from changes. +- **Python — runner.py**: `_schema` on `MigrationRunner`, `.schema()` builder, `_create_ddl()` helper, schema-aware `_operation_to_ddl()`, `_ddl_for_change()`, `_introspect_schema()`, `_get_tables()`, `_get_columns()`, `_apply_meta_extras()`, `_ensure_m2m_table()`. +- **Python — query layer**: `QuerySet.schema()` method adds `("schema", name)` op to ops list. Rust `build_plan()` in `ryx-python/src/plan.rs` handles `"schema"` op via `node.with_schema()`. +- **Python — CLI**: `--schema` flag on `migrate`, `sqlmigrate`, `inspectdb` commands in `ryx-python/ryx/__main__.py`, `ryx-python/ryx/cli/commands/migrate.py`, `sqlmigrate.py`, `inspectdb.py`. +- **Rust unit tests** — **30 new multi-schema tests added** (75 unit total): + - migration.rs: 7 multi-schema diff tests (CreateSchema detection, same table different schemas, empty/noop/mixed) + - ddl.rs: 20 schema-qualified DDL tests (create_schema, create_table, alter, drop, index, FK, constraint, MySQL/SQLite ignore, backward compat) + - operations.rs: 7 operation schema tests (all variants) + - compilr.rs: 12 schema query tests (SELECT/COUNT/DELETE/UPDATE/INSERT/JOIN with schema, MySQL/SQLite ignore, plan hash differential) +- **Python unit tests** — **42 new tests added**: + - test_migration_state.py: 11 tests (diff_states composite key, CreateSchema detection, serialization) + - test_migration_ddl.py: 17 tests (create_schema, qualified names, all DDL methods, backward compat) + - test_migration_autodetect.py: 10 tests (operation schema fields, to_python, apply_migration_to_state, AddField schema inheritance) +- **Rust integration test**: `ryx-rs/tests/multi_schema_test.rs` — full multi-schema pipeline: migrate tenant1, migrate tenant2, verify table isolation across schemas, insert/query data independently, cleanup. Uses `PG_TEST_URL` env var (defaults to `postgres://einswilli@localhost/ryx_integration_test`). Passes. +- **Python integration test**: `ryx-python/tests/integration/test_multi_schema.py` — same pipeline, runs in subprocess to avoid conftest's SQLite pool. Overrides `setup_database` and `clean_tables` fixtures. Passes. + +### In Progress +- (none — feature complete) + +### Blocked +- Python `_handle_no_migration_files` interactive prompt blocks automated migration testing (no `live` flag). Tests work around it by monkeypatching `input()`. + +## Key Decisions +- **Schema at query/migration time, not on model**: Models stay agnostic → one model works in N schemas without duplication. +- **Empty string = no schema = backward compat**: All existing code continues to work unchanged. Schema qualification only activates when explicitly set. +- **`Option` vs `Option` in QueryNode**: `Option` for interned performance, `None` = no qualification. +- **Composite key `(schema, name)` everywhere**: Enables correct diff across schemas. No schema = empty string matches "no schema" (no `CreateSchema` emitted). +- **`CREATE SCHEMA IF NOT EXISTS` is automatic**: Runner creates schema before tables when schema is non-empty and backend supports schemas. Not part of migration files (idempotent). +- **Per-op schema override**: Each operation can carry its own schema, allowing mixed-schema migration files (unlikely but supported). +- **Temporary DDLGenerator per change**: Used in `generate()`, `apply_files()`, `plan_files()` to switch schema context. Acceptable cost since migration ops are rare. +- **Python QuerySet delegates schema to Rust FFI**: `("schema", name)` op is stored in ops list and handled by Rust `build_plan()`. This means all SQL compilation is schema-aware with zero additional Python overhead. +- **Integration tests run in subprocess for Python**: The `ryx` package auto-initializes the pool from `ryx.toml` at import time. Python's conftest also inits a SQLite pool. PG tests must run separately via subprocess with `RYX_AUTO_INITIALIZE=0`. + +## Next Steps +1. Add `live=True` flag to Python `MigrationRunner` to skip interactive prompt in tests (feature gap). +2. Documentation update: multi-schema section in migrations.mdx, `.schema()` API reference. + +## Critical Context +- **Rust integration tests**: 76 unit + 2 integration (SQLite full pipeline + PG multi-schema). +- **Python integration tests**: 42 unit + 1 integration (PG multi-schema via subprocess). +- `diff_states()` matches tables by `(schema, name)` composite key. Two tables with same name in different schemas are treated as different tables. `CreateSchema` is emitted for schemas with non-empty string. +- `DDLGenerator.qn()` produces `"schema"."table"` when schema is non-empty and backend supports schemas. For MySQL/SQLite, `supports_schemas()` → `false` eliminates qualification at compile time. +- `QueryNode.with_schema(schema)` interns the schema name as `Symbol`. The compiler's `write_table_ref()` checks `node.backend.supports_schemas()` at compile time. +- Python `QuerySet.schema()` adds an op that the Rust FFI `build_plan()` converts to `node.with_schema()`. The same Rust compiler handles qualification. +- `__main__.py` has two parser code paths: the old argparse-based `_build_parser()` and the new registry-based `build_parser()`. `--schema` was added only to `_build_parser()` for now. +- `inspectdb.py` hardcodes `table_schema = 'public'` for PostgreSQL introspection (the `--schema` flag passes through to the runner layer). +- Integration test requires a running PostgreSQL on localhost:5432 with user `einswilli` (no password) and a `ryx_integration_test` database. +- Python `ryx` package has `ryx/__init__.py:_auto_setup()` that reads `ryx.toml` and initializes the pool at `import ryx` time. Set `RYX_AUTO_INITIALIZE=0` to prevent this. +- Python conftest adds `setup_database` (session-scoped, SQLite) and `clean_tables` (autouse, async) to all integration tests. PG tests must override both. + +## Relevant Files +- **Rust modified**: + - `ryx-query/src/ast.rs`: `QueryNode.schema` + `with_schema()`. + - `ryx-query/src/compiler/compilr.rs`: `write_table_ref()`, `write_join_table_ref()`, all compile functions updated, plan hash includes schema. + - `ryx-query/src/backend.rs`: `supports_schemas()` const fn. + - `ryx-rs/src/migration.rs`: core types, `diff_states()`, introspection, test helpers, 7 new schema diff tests. + - `ryx-rs/src/migration/ddl.rs`: `DDLGenerator.in_schema()`, `qn()`, `create_schema()`, `generate()` zeroth pass, 20 new DDL schema tests. + - `ryx-rs/src/migration/operations.rs`: `CreateSchema` variant, `schema` on all ops, `schema()` accessor, 7 new operation schema tests. + - `ryx-rs/src/migration/autodetect.rs`: `ModelEntry.schema`, `build_target()` schema, `changes_to_operations()` CreateSchema. + - `ryx-rs/src/migration/runner.rs`: `FileRunner.schema`, `.schema()` builder, schema-aware run/plan/apply. + - `ryx-rs/src/queryset.rs`: `.schema()` builder method. + - `ryx-rs/src/main.rs`: `--schema` on Migrate/Sqlmigrate. +- **Python modified**: + - `ryx-python/ryx/migrations/state.py`: `TableState.schema`, `ChangeKind.CREATE_SCHEMA`, `SchemaChange.schema`, `diff_states()` composite key, `to_json()`/`from_json()` schema-aware. + - `ryx-python/ryx/migrations/ddl.py`: `DDLGenerator.schema`, `_qn()`, `create_schema()`, all DDL methods qualified, `generate_schema_ddl()` per-table schema. + - `ryx-python/ryx/migrations/autodetect.py`: `schema` on all operation types, `to_python()` includes schema, `apply_migration_to_state()` preserves schema. + - `ryx-python/ryx/migrations/runner.py`: `_schema`, `.schema()` builder, `_create_ddl()`, all internal methods schema-aware. + - `ryx-python/ryx/queryset.py`: `.schema()` method. + - `ryx-python/src/plan.rs`: `"schema"` op handler. + - `ryx-python/ryx/__main__.py`: `--schema` on migrate/sqlmigrate/inspectdb. + - `ryx-python/ryx/cli/commands/migrate.py`: `--schema` flag + pass to runner. + - `ryx-python/ryx/cli/commands/sqlmigrate.py`: `--schema` flag + DDLGenerator schema. + - `ryx-python/ryx/cli/commands/inspectdb.py`: `--schema` flag + schema-aware introspection. +- **Test files**: + - `ryx-rs/src/migration/operations.rs`: 7 new schema tests. + - `ryx-python/tests/unit/test_migration_state.py`: 11 new tests. + - `ryx-python/tests/unit/test_migration_ddl.py`: 17 new tests. + - `ryx-python/tests/unit/test_migration_autodetect.py`: 10 new tests. + - `ryx-rs/tests/multi_schema_test.rs`: PG multi-schema integration test. + - `ryx-python/tests/integration/test_multi_schema.py`: Python PG multi-schema integration test. diff --git a/docs/doc/advanced/cli.mdx b/docs/doc/advanced/cli.mdx index 609fe61..e0d20bf 100644 --- a/docs/doc/advanced/cli.mdx +++ b/docs/doc/advanced/cli.mdx @@ -4,122 +4,283 @@ sidebar_position: 10 # CLI -Ryx includes a command-line interface for common database tasks. +Ryx ships a management CLI for migrations, introspection, shell access, native +database shells, destructive flushes, and version checks. ```bash -python -m ryx [options] +python -m ryx [global-options] [command-options] +ryx [global-options] [command-options] ``` -## Commands +Global options: -### migrate +| Flag | Purpose | +|---|---| +| `--url`, `-u` | Database URL. Overrides environment/config values | +| `--settings`, `-s` | Settings module name. Defaults to `ryx_settings` | +| `--verbose`, `-v` | Enable verbose output | +| `--debug` | Enable debug mode | -Apply migrations to the database: +The command registry is plugin-aware, so additional commands can be registered +by Ryx CLI plugins. + +## Configuration Resolution + +The CLI resolves configuration from: + +1. Global CLI flags +2. Environment variables +3. A settings module +4. Config files in the current working directory + +Supported config file names: + +```text +ryx.yaml +ryx.yml +ryx.toml +ryx.json +``` + +Environment variables: ```bash -python -m ryx migrate \ - --url postgres://user:pass@localhost/mydb \ - --models myapp.models +export RYX_DATABASE_URL="postgres://user:pass@localhost/app" +export RYX_DB_REPLICA_URL="postgres://user:pass@localhost/app_replica" +export RYX_DB_LOGS_URL="sqlite:///logs.db" ``` -### makemigrations +Example TOML: -Generate migration files: +```toml +[urls] +default = "postgres://user:pass@localhost/app" +replica = "postgres://user:pass@localhost/app_replica" -```bash -python -m ryx makemigrations \ - --models myapp.models \ - --dir migrations/ +[pool] +max_conn = 10 +min_conn = 1 +connect_timeout = 30 +idle_timeout = 600 +max_lifetime = 1800 + +[models] +files = ["myapp.models"] + +[migrations] +dir = "migrations" +``` + +## `makemigrations` -# Check mode — exit 1 if unapplied changes +Detect model changes and generate migration files. + +```bash +python -m ryx makemigrations --models myapp.models +python -m ryx makemigrations --models myapp.models --name add_posts python -m ryx makemigrations --models myapp.models --check ``` -### showmigrations +Options: + +| Flag | Purpose | +|---|---| +| `--models MODULE` | Dotted module path containing models; can also come from config | +| `--dir DIR` | Migrations directory. Defaults to `migrations` | +| `--alias ALIAS` | Generate migrations in an alias-specific subdirectory | +| `--name NAME` | Override the generated migration name slug | +| `--check` | Exit with status `1` if changes are detected | +| `--squash` | Reserved flag for squashing multiple migrations | -Show migration status: +Use `--check` in CI to fail when models and migration files drift. + +## `migrate` + +Apply pending migrations. ```bash -python -m ryx showmigrations \ - --url postgres://user:pass@localhost/mydb \ - --dir migrations/ +python -m ryx --url postgres://user:pass@localhost/app migrate --models myapp.models +python -m ryx migrate --models myapp.models --dry-run +python -m ryx migrate --models myapp.models --database replica +python -m ryx migrate --models myapp.models --schema tenant_1 ``` -### sqlmigrate +Options: + +| Flag | Purpose | +|---|---| +| `--dry-run` | Print SQL without executing | +| `--models MODULE` | Dotted module path containing models | +| `--dir DIR` | Migrations directory. Defaults to `migrations` | +| `--plan` | Show migration plan without executing | +| `--database ALIAS` | Run migrations for one database alias | +| `--no-interactive` | Fail if no migration files are found; useful in CI | +| `--schema SCHEMA` | PostgreSQL schema for multi-schema migrations | -Print SQL for a specific migration: +`migrate` initializes Ryx with the resolved URL mapping before running the +migration runner. + +## `showmigrations` + +List migration files and whether they are applied. ```bash -python -m ryx sqlmigrate 0001_initial --dir migrations/ +python -m ryx showmigrations --dir migrations +python -m ryx showmigrations --unapplied ``` -### flush +Options: + +| Flag | Purpose | +|---|---| +| `--dir DIR` | Migrations directory | +| `--unapplied` | Show only unapplied migrations | -Delete all data from all tables: +## `sqlmigrate` + +Print SQL for one migration without executing it. ```bash -python -m ryx flush \ - --models myapp.models \ - --url postgres://user:pass@localhost/mydb \ - --yes +python -m ryx sqlmigrate 0001_initial --dir migrations +python -m ryx sqlmigrate 0001_initial --backends postgres,mysql,sqlite +python -m ryx sqlmigrate 0001_initial --schema tenant_1 ``` +Options: + +| Flag | Purpose | +|---|---| +| `name` | Migration name, for example `0001_initial` | +| `--dir DIR` | Migrations directory. Defaults to `migrations` | +| `--backends LIST` | Comma-separated backend filter: `postgres,mysql,sqlite` | +| `--schema SCHEMA` | PostgreSQL schema | + +## `flush` + +Delete all rows from all model tables. + +```bash +python -m ryx --url sqlite:///app.db flush --models myapp.models +python -m ryx --url sqlite:///app.db flush --models myapp.models --yes +``` + +Options: + +| Flag | Purpose | +|---|---| +| `--models MODULE` | Required dotted module path containing models | +| `--yes` | Skip the confirmation prompt | +| `--force` | Alias for `--yes` | + :::warning -This deletes ALL data. Use `--yes` to skip the confirmation prompt. +`flush` is destructive. It issues `DELETE FROM ""` for every loaded +managed model table. Use `--yes` or `--force` only in controlled environments. ::: -### shell +## `shell` -Interactive Python shell with ORM pre-loaded: +Start an interactive Python shell. ```bash -python -m ryx shell \ - --url postgres://user:pass@localhost/mydb \ - --models myapp.models +python -m ryx shell --models myapp.models +python -m ryx shell --query "await Post.objects.count()" +python -m ryx shell --notebook ``` -### dbshell +Options: + +| Flag | Purpose | +|---|---| +| `--models MODULE` | Pre-import models from a module | +| `--query`, `-q` | Execute a query and print results | +| `--ipyazthon` | Current registered flag for IPython mode | +| `--notebook` | Launch Jupyter notebook instead of a shell | + +:::note +The code currently registers the IPython flag as `--ipyazthon` while execution +checks `ipython`. Until that implementation typo is fixed, prefer the plain +shell or notebook mode. +::: + +## `dbshell` -Connect to the database with the native CLI client: +Open the native database command-line client. ```bash -python -m ryx dbshell --url postgres://user:pass@localhost/mydb -# Opens psql +python -m ryx --url postgres://user:pass@localhost/app dbshell +python -m ryx --url mysql://user:pass@localhost/app dbshell +python -m ryx --url sqlite:///app.db dbshell +python -m ryx --url sqlite:///app.db dbshell -c ".tables" ``` -### inspectdb +Ryx selects the native client from the URL scheme, such as `psql`, `mysql`, or +`sqlite3`. The matching CLI tool must be installed on your system. -Introspect an existing database and generate model stubs: +Options: -```bash -# All tables -python -m ryx inspectdb --url postgres://user:pass@localhost/mydb +| Flag | Purpose | +|---|---| +| `--command`, `-c CMD` | Execute one native database command and exit | -# Specific table -python -m ryx inspectdb --url postgres://user:pass@localhost/mydb --table users +## `inspectdb` + +Introspect an existing database and print Ryx model stubs. + +```bash +python -m ryx --url postgres://user:pass@localhost/app inspectdb +python -m ryx --url postgres://user:pass@localhost/app inspectdb --table users +python -m ryx --url postgres://user:pass@localhost/app inspectdb --schema tenant_1 -o models.py ``` -### version +Options: + +| Flag | Purpose | +|---|---| +| `--table TABLE` | Introspect one table | +| `--output`, `-o FILE` | Write generated model stubs to a file | +| `--schema SCHEMA` | Schema to inspect. Defaults to `public` | + +For PostgreSQL/MySQL, Ryx reads `information_schema`. For SQLite, it falls back +to `sqlite_master`. + +## `version` + +Print the installed Ryx version. ```bash python -m ryx version +python -m ryx version --verbose ``` -## Configuration +Options: + +| Flag | Purpose | +|---|---| +| `--verbose`, `-v` | Include Rust core version information when available | -The CLI reads configuration from: +## Plugins -1. **CLI flags** — `--url`, `--models`, `--dir` -2. **Environment variable** — `RYX_DATABASE_URL` -3. **Settings module** — `ryx_settings.py` in your project +CLI plugins can register extra commands. A plugin implements the `Plugin` +interface and is loaded either from settings/config or Python entry points. ```python -# ryx_settings.py -DATABASE_URL = "postgres://user:pass@localhost/mydb" -MODELS = ["myapp.models"] -MIGRATIONS_DIR = "migrations/" +from ryx.cli.plugins import Plugin + +class MyPlugin(Plugin): + name = "my-plugin" + + def register_commands(self, registry): + registry["mycommand"] = MyCommand ``` -## Next Steps +Use plugins when you need project-specific management commands without +patching Ryx itself. + +## Operational Notes -→ **[Reference](/reference/api-reference)** — Complete API documentation +- Use `RYX_LOG_LEVEL=debug` for verbose query/runtime diagnostics. +- Use `NO_COLOR=1` to disable ANSI colors. +- Use `--no-interactive` and `makemigrations --check` in CI. +- Prefer `--database` for `migrate`; `--alias` belongs to `makemigrations`. +- Use config files for multi-database projects instead of repeating long URL + arguments in every command. diff --git a/docs/doc/advanced/configuration-and-routing.mdx b/docs/doc/advanced/configuration-and-routing.mdx new file mode 100644 index 0000000..35b8bd9 --- /dev/null +++ b/docs/doc/advanced/configuration-and-routing.mdx @@ -0,0 +1,186 @@ +--- +sidebar_position: 9 +--- + +# Configuration and Routing + +Ryx supports explicit setup, import-time auto-initialization, multiple database +aliases, per-model database metadata, and a global database router. + +## Explicit Setup + +```python +import ryx + +await ryx.setup("sqlite:///app.db") +``` + +For multiple databases, pass a mapping: + +```python +await ryx.setup({ + "default": "postgres://user:pass@localhost/app", + "replica": "postgres://user:pass@localhost/app_replica", + "analytics": "sqlite:///analytics.db", +}) +``` + +Pool options: + +```python +await ryx.setup( + {"default": "sqlite:///app.db"}, + max_connections=10, + min_connections=1, + connect_timeout=30, + idle_timeout=600, + max_lifetime=1800, +) +``` + +## Auto Initialization + +On import, Ryx attempts auto-initialization unless disabled: + +```bash +export RYX_AUTO_INITIALIZE=0 +``` + +Disabling values are `0`, `false`, `n`, and `no`. + +Auto-init reads database URLs from: + +```bash +export RYX_DATABASE_URL="postgres://user:pass@localhost/app" +export RYX_DB_REPLICA_URL="postgres://user:pass@localhost/app_replica" +``` + +`RYX_DATABASE_URL` becomes the `default` alias. `RYX_DB__URL` becomes the +lowercase alias name. + +## Config Files + +Ryx looks for the first existing file in the current directory: + +```text +ryx.yaml +ryx.yml +ryx.toml +ryx.json +``` + +Example `ryx.toml`: + +```toml +[urls] +default = "postgres://user:pass@localhost/app" +replica = "postgres://user:pass@localhost/app_replica" +logs = "sqlite:///logs.db" + +[pool] +max_conn = 10 +min_conn = 1 +connect_timeout = 30 +idle_timeout = 600 +max_lifetime = 1800 + +[models] +files = ["myapp.models", "billing.models"] + +[migrations] +dir = "migrations" +``` + +The same structure can be represented as YAML or JSON. + +## Model-Level Database Alias + +Set `Meta.database` to bind a model to an alias by default: + +```python +class AuditLog(Model): + event = CharField(max_length=200) + + class Meta: + table_name = "audit_logs" + database = "logs" +``` + +Reads and writes fall back to `Meta.database` unless `.using()` or a router +returns an alias first. + +## Query-Level Routing + +Use `.using(alias)` for one query: + +```python +logs = await AuditLog.objects.using("logs").filter(event__icontains="login") +posts = await Post.objects.using("replica").filter(active=True) +``` + +Use `.schema(schema)` for PostgreSQL multi-schema queries: + +```python +tenant_posts = await Post.objects.schema("tenant_42").filter(active=True) +``` + +## Global Router + +```python +from ryx.router import BaseRouter, set_router + +class ReadReplicaRouter(BaseRouter): + def db_for_read(self, model, **hints): + if getattr(model._meta, "database", None): + return model._meta.database + return "replica" + + def db_for_write(self, model, **hints): + return getattr(model._meta, "database", None) or "default" + + def allow_migrate(self, db, app_label, model_name): + return True + +set_router(ReadReplicaRouter()) +``` + +Router methods return an alias string or `None`. Returning `None` tells Ryx to +continue fallback resolution. + +## Resolution Order + +Read operations resolve aliases in this order: + +1. QuerySet `.using(alias)` +2. `Router.db_for_read(model)` +3. `Model.Meta.database` +4. `"default"` + +Write operations resolve aliases in this order: + +1. Explicit `save(using=alias)` or QuerySet `.using(alias)` where supported +2. `Router.db_for_write(model)` +3. `Model.Meta.database` +4. `"default"` + +## Introspection Helpers + +```python +ryx.is_connected("default") +ryx.list_aliases() +ryx.pool_stats() +``` + +Use these helpers for startup diagnostics and health checks. + +## CLI Integration + +The CLI shares the same config discovery rules. For multi-database work: + +```bash +python -m ryx migrate --models myapp.models --database default +python -m ryx makemigrations --models myapp.models --alias logs +``` + +Use `--database` for applying migrations to one alias. Use `--alias` when +generating alias-specific migration files. diff --git a/docs/doc/advanced/index.mdx b/docs/doc/advanced/index.mdx index 23e33d3..96f1e03 100644 --- a/docs/doc/advanced/index.mdx +++ b/docs/doc/advanced/index.mdx @@ -15,6 +15,7 @@ Deep-dive topics for production-ready applications. - **[Caching](./caching)** — Query result caching - **[Custom Lookups](./custom-lookups)** — Extend the query API - **[Sync/Async](./sync-async)** — Bridge between sync and async code -- **[Multi-Databases](./multi-db)** - Multi-Database Support +- **[Multi-Databases](./multi-db)** — Multi-Database Support +- **[PostgreSQL Multi-Schema](./postgres-multi-schema)** — Schema-per-tenant data isolation - **[Raw SQL](./raw-sql)** — Escape hatch for complex queries - **[CLI](./cli)** — Command-line management commands diff --git a/docs/doc/advanced/multi-db.mdx b/docs/doc/advanced/multi-db.mdx index c03934f..e8f11c8 100644 --- a/docs/doc/advanced/multi-db.mdx +++ b/docs/doc/advanced/multi-db.mdx @@ -6,6 +6,8 @@ description: Learn how to route queries across multiple databases in Ryx. Ryx supports routing queries across multiple databases, allowing you to separate read and write workloads, split data across different servers, or use a dedicated database for specific models. +> **Also see**: [PostgreSQL Multi-Schema](./postgres-multi-schema) for schema-per-tenant isolation within a single database. + ## Configuration To enable multi-database support, provide a dictionary of URLs to `ryx_core.setup` instead of a single string. Each key in the dictionary serves as an **alias** for that database pool. diff --git a/docs/doc/advanced/postgres-multi-schema.mdx b/docs/doc/advanced/postgres-multi-schema.mdx new file mode 100644 index 0000000..c191a90 --- /dev/null +++ b/docs/doc/advanced/postgres-multi-schema.mdx @@ -0,0 +1,160 @@ +--- +sidebar_position: 12 +title: PostgreSQL Multi-Schema (Schema-per-Tenant) +description: Isolate tenant data using PostgreSQL schemas with Ryx. +--- + +Ryx supports PostgreSQL **database schemas** (`CREATE SCHEMA`) to isolate data per tenant, environment, or module — all within a single database connection. + +Each schema has its own tables, indexes, and data. Queries and migrations are scoped to a schema via the `.schema()` builder or `--schema` CLI flag. + +## When to Use + +- **Schema-per-tenant SaaS**: each tenant gets its own schema (`tenant1`, `tenant2`, …) +- **Environment isolation**: `staging` and `production` schemas in the same DB +- **Logical data domains**: `analytics`, `logging`, `billing` as separate schemas + +## Migration with a Schema + +### Python + +Use the `--schema` flag to target a specific schema: + +```bash +python -m ryx migrate \ + --url postgres://user:pass@localhost/mydb \ + --models myapp.models \ + --schema tenant1 +``` + +Or set the schema programmatically via the `MigrationRunner`: + +```python +from ryx.migrations import MigrationRunner + +runner = MigrationRunner( + [Author, Post], + schema="tenant1", +) +await runner.migrate() +``` + +Chained builder style: + +```python +runner = MigrationRunner([Author, Post]).schema("tenant1") +await runner.migrate() +``` + +### Rust + +Use `.schema()` on the `FileRunner` builder: + +```rust +use ryx_rs::migration::FileRunner; + +FileRunner::new() + .model::() + .model::() + .schema("tenant1") + .run().await?; +``` + +The DDL generator also accepts schema directly: + +```rust +use ryx_rs::migration::DDLGenerator; +use ryx_query::Backend; + +let ddl = DDLGenerator::new(Backend::PostgreSQL) + .in_schema("tenant1"); +let sql = ddl.create_table(&table_state); +// → CREATE TABLE IF NOT EXISTS "tenant1"."posts" ( ... ) +``` + +## Querying with a Schema + +### Python + +```python +# All queries in this schema +posts = await Post.objects.schema("tenant1").filter(active=True) + +# Chained with other methods +tenants_posts = await ( + Post.objects + .schema("tenant1") + .filter(views__gte=100) + .order_by("-created_at") +) +``` + +### Rust + +```rust +let posts: Vec = Post::objects() + .schema("tenant1") + .filter("active", true) + .all().await?; +``` + +## Creating Schemas + +Ryx auto-creates schemas during migration if they don't exist. The `CREATE SCHEMA IF NOT EXISTS` statement is issued before any table DDL when a schema is specified. + +You can also create schemas manually: + +### Python + +```python +from ryx.migrations.ddl import DDLGenerator + +gen = DDLGenerator("postgres") +sql = gen.create_schema("tenant1") +# → CREATE SCHEMA IF NOT EXISTS "tenant1" +``` + +### Rust + +```rust +use ryx_rs::migration::DDLGenerator; + +let ddl = DDLGenerator::new(ryx_query::Backend::PostgreSQL); +let sql = ddl.create_schema("tenant1"); +// → CREATE SCHEMA IF NOT EXISTS "tenant1" +``` + +## Multi-Tenant Example + +Here's a complete schema-per-tenant pattern with six tenants, each getting `ms_authors` and `ms_posts` tables: + +```python +tenants = ["tenant1", "tenant2", "tenant3", + "tenant4", "tenant5", "tenant6"] + +for tenant in tenants: + runner = MigrationRunner([Author, Post], schema=tenant) + await runner.migrate() + +# Query tenant2's data +posts = await Post.objects.schema("tenant2").filter(active=True) +``` + +## Migration Tracking + +Migration state is tracked per-schema independently. Each schema has its own `ryx_migrations` table recording which files have been applied to that schema. + +```bash +# Check status for a specific schema +python -m ryx showmigrations --schema tenant1 +``` + +## Backend Support + +PostgreSQL schemas are a **PostgreSQL-only** feature. MySQL and SQLite do not support database schemas — the `--schema` flag and `.schema()` method are silently ignored on those backends. + +## Next Steps + +- **[Multi-Database Support](./multi-db)** — Route queries across multiple database connections +- **[Migrations](/core-concepts/migrations)** — Full migration reference +- **[CLI Reference](./cli)** — All management commands diff --git a/docs/doc/core-concepts/migrations.mdx b/docs/doc/core-concepts/migrations.mdx index f9cb4fa..6d62a63 100644 --- a/docs/doc/core-concepts/migrations.mdx +++ b/docs/doc/core-concepts/migrations.mdx @@ -4,7 +4,17 @@ sidebar_position: 5 # Migrations -Ryx can introspect your database, detect schema changes, and generate DDL automatically. +Ryx can introspect your database, detect schema changes, and generate DDL automatically — with full multi-database routing and colored CLI output. + +## CLI in Action + +The CLI uses ANSI colors on supported terminals (macOS, Linux). Colors auto-disable when `NO_COLOR` is set or output is piped: + +``` + [ryx] ✓ 0001_initial ← bold blue prefix, green check + [ryx] ⚠ No migration files exist ← yellow warning + [ryx] Skipped database blog ← yellow "Skipped", magenta alias +``` ## Two Approaches @@ -47,15 +57,100 @@ python -m ryx showmigrations \ --dir migrations/ ``` -## How It Works +## Migration File Format + +Generated migration files are clean, multi-line Python with **model class references** for automatic database routing: + +```python +from myapp.models import Author, Post +from ryx.migrations.autodetect import CreateTable, AddField, ColumnState + +Migration = [ + CreateTable( + table='authors', + columns=[ + ColumnState(name='id', db_type='INTEGER', primary_key=True, unique=True), + ColumnState(name='name', db_type='VARCHAR(100)'), + ], + model=Author, + ), + CreateTable( + table='posts', + columns=[ + ColumnState(name='id', db_type='INTEGER', primary_key=True, unique=True), + ColumnState(name='title', db_type='VARCHAR(200)'), + ColumnState(name='author_id', db_type='INTEGER'), + ], + model=Post, + ), + AddField( + table='posts', + column=ColumnState(name='views', db_type='INTEGER', nullable=False), + model=Post, + ), +] +``` + +Key features: +- Each operation embeds the **Model class** (`model=Author`) so the runner knows which database it belongs to via `model._meta.database` +- Models are imported at the top of the file (`from myapp.models import Author`) +- Output is indented and readable, one argument per line +- Backward compatible — legacy files without `model=` still work + +## File Discovery + +Migration files are discovered **recursively** under the migrations directory. All files matching `[0-9]*.py` are found, sorted globally by numeric prefix: + +``` +migrations/ +├── 0001_initial.py +├── 0002_add_views.py +├── blog/ +│ ├── 0001_blog_tables.py +│ └── 0002_add_tags.py +└── shop/ + └── 0001_shop_tables.py +``` + +Files in subdirectories are found automatically — no per-alias configuration needed. + +## Multi-Database Routing + +When using multiple databases, each operation knows its target database from the embedded Model class. The runner: + +1. Discovers **all** migration files recursively +2. For each database alias, filters operations whose `model._meta.database` matches +3. Applies only relevant operations per alias +4. Tracks applied state as `alias|stem` in `ryx_migrations` +```bash +# Apply only to the "blog" alias +python -m ryx migrate --database blog + +# Target a specific PostgreSQL schema (schema-per-tenant) +python -m ryx migrate --schema tenant1 + +# Non-interactive mode (for CI/CD) +python -m ryx migrate --no-interactive ``` -1. Introspect live DB schema → SchemaState (current) -2. Build target from Models → SchemaState (desired) -3. Diff the two states → List of changes -4. Generate DDL → CREATE TABLE, ALTER COLUMN, etc. -5. Execute → Apply to database + +## Interactive Fallback + +When no migration files exist yet, `ryx migrate` offers an interactive menu: + ``` + [ryx] ⚠ No migration files exist for database default + 3 model(s) are not yet tracked. + + L)ive DDL — apply changes directly (development only) + A)uto-generate migration files, then migrate + M)anual — run ryx makemigrations first + S)kip this database for now + + [ryx] Choice [S]: +``` + +Use `--no-interactive` to skip this prompt in scripts or CI/CD (exits with a hint instead). ## Migration Tracking @@ -64,9 +159,11 @@ Ryx creates a `ryx_migrations` table to track applied migrations: | Column | Type | |---|---| | `id` | INTEGER | -| `name` | TEXT | +| `name` | TEXT (`alias|stem` format) | | `applied_at` | TIMESTAMP | +Applied entries use `alias|stem` (e.g. `default|0001_initial`, `blog|0001_blog_tables`). Legacy bare-stem entries are still recognized for backward compatibility. + ## What Migrations Handle - Creating and dropping tables @@ -97,12 +194,13 @@ print(gen.add_column("posts", column_state)) ## Backend Differences | Feature | PostgreSQL | MySQL | SQLite | -|---|---|---|---| +|---|---|---|---|---| | `ALTER COLUMN` | Yes | Yes | No (recreate table) | | Native UUID | Yes | No | No | | `SERIAL` | Yes | No | No | | `JSONB` | Yes | No | No | | Array types | Yes | No | No | +| **Database schemas** | **`"schema"."table"`** | No | No | ## Next Steps diff --git a/docs/doc/internals/performance.mdx b/docs/doc/internals/performance.mdx index c7da52c..c563d2b 100644 --- a/docs/doc/internals/performance.mdx +++ b/docs/doc/internals/performance.mdx @@ -4,7 +4,8 @@ sidebar_position: 4 # Performance Optimizations -Ryx is engineered for extreme performance, with a primary target of **1-2 $\mu$s overhead** for query construction and row decoding. This is achieved by eliminating common abstractions that introduce runtime overhead. +Ryx is engineered for low overhead in query construction and row decoding. This +is achieved by eliminating common abstractions that introduce runtime overhead. ## 1. Enum Dispatch vs. Dynamic Dispatch @@ -58,8 +59,8 @@ Instead of duplicating column names for every row, Ryx separates the **Structure ### Performance Impact | Approach | Allocations per Row | Memory Layout | Complexity | |---|---|---|---| -| `HashMap` | $\sim$10-20 | Scattered | $O(\text{cols})$ | -| **RowView** | **1 (The View itself)** | Linear / Contiguous | $O(1)$ | +| `HashMap` | around 10-20 | Scattered | proportional to column count | +| **RowView** | **1 view object** | Linear / contiguous | constant lookup after mapping | This reduces allocator pressure by orders of magnitude and significantly improves cache locality. diff --git a/docs/doc/intro.mdx b/docs/doc/intro.mdx index f2106c1..cbfc5c9 100644 --- a/docs/doc/intro.mdx +++ b/docs/doc/intro.mdx @@ -9,12 +9,12 @@ import { Badge } from '@site/src/components/Badge'; # Ryx ORM
- Django-style Python ORM. Powered by Rust.
- Ergonomic query API. Async-native. Zero GIL blocking. Compiled performance. + Django-style ORM for Python and Rust. Powered by a shared Rust SQL engine.
+ Async-native Python API, Rust-native API, compiled query planning, and sqlx-backed execution.
- v0.1.0 + v0.1.x Python 3.10+ PostgreSQL MySQL @@ -23,7 +23,7 @@ import { Badge } from '@site/src/components/Badge'; ```python import ryx -from ryx import Model, CharField, IntField, Q, Count, Sum +from ryx import Model, BooleanField, CharField, IntField, Q class Post(Model): title = CharField(max_length=200) @@ -46,8 +46,8 @@ posts = await ( | | Django ORM | SQLAlchemy | **Ryx** | |---|---|---|---| | **API** | Ergonomic | Verbose | **Ergonomic** | -| **Runtime** | Sync Python | Async Python | **Async Rust** | -| **GIL blocking** | Yes | Yes | **Zero** | +| **Runtime** | Sync Python | Async Python | **Async Python + Rust execution** | +| **Query engine** | Python | Python | **Rust compiler + sqlx backend** | | **Backends** | All | All | **PG · MySQL · SQLite** | | **Migrations** | Built-in | Alembic | **Built-in** | @@ -57,7 +57,7 @@ posts = await ( @@ -113,20 +113,20 @@ posts = await ( ## Quick Start ```bash -pip install maturin -maturin develop # compile Rust + install +pip install ryx ``` ```python import asyncio, ryx from ryx import Model, CharField +from ryx.migrations import MigrationRunner class Article(Model): title = CharField(max_length=200) async def main(): await ryx.setup("sqlite:///app.db") - await ryx.migrate([Article]) + await MigrationRunner([Article]).migrate() await Article.objects.create(title="Hello Ryx") print(await Article.objects.all()) diff --git a/docs/doc/reference/api-reference.mdx b/docs/doc/reference/api-reference.mdx index 2b1f239..b4292fa 100644 --- a/docs/doc/reference/api-reference.mdx +++ b/docs/doc/reference/api-reference.mdx @@ -4,32 +4,298 @@ sidebar_position: 2 # API Reference -Complete public API surface of Ryx. +This page documents the public Python API exposed by `ryx`. It is aligned with +the current codebase: `ryx-python/ryx/__init__.py`, `models.py`, +`queryset.py`, `fields.py`, `transaction.py`, `cache.py`, `signals.py`, +`router.py`, and the PyO3 extension stub. -## Setup & Connection +## Setup and Connection Pools ```python import ryx -await ryx.setup(url, max_connections=10, min_connections=1, connect_timeout=30, idle_timeout=600, max_lifetime=1800) -ryx.is_connected() # → bool -ryx.pool_stats() # → dict with pool statistics +await ryx.setup( + "sqlite:///app.db", + max_connections=10, + min_connections=1, + connect_timeout=30, + idle_timeout=600, + max_lifetime=1800, +) +``` + +`setup()` accepts either a single URL or a mapping of aliases to URLs: + +```python +await ryx.setup({ + "default": "postgres://user:pass@localhost/app", + "replica": "postgres://user:pass@localhost/app_replica", + "logs": "sqlite:///logs.db", +}) ``` +Connection helpers: + +| Function | Purpose | +|---|---| +| `await ryx.setup(urls, **pool_options)` | Initialize one or more connection pools | +| `ryx.is_connected(alias="default")` | Return whether a pool alias is initialized | +| `ryx.list_aliases()` | Return configured database aliases | +| `ryx.pool_stats()` | Return pool statistics from the Rust backend | + +Auto-initialization runs on import unless `RYX_AUTO_INITIALIZE` is one of +`0`, `false`, `n`, or `no`. It reads `RYX_DATABASE_URL`, +`RYX_DB__URL`, and the first config file found among `ryx.yaml`, +`ryx.yml`, `ryx.toml`, and `ryx.json`. + +Logging is controlled by `RYX_LOG_LEVEL`. Use `NO_LOG`, `OFF`, or `SILENT` to +disable Ryx logging. + ## Models ```python -from ryx import Model, Index, Constraint +from ryx import Model, CharField, IntField, Index, Constraint + +class Post(Model): + title = CharField(max_length=200, db_index=True) + views = IntField(default=0) -class MyModel(Model): class Meta: - table_name = "custom_name" - ordering = ["-created_at"] - unique_together = [("field1", "field2")] - indexes = [Index(fields=["field"], name="idx")] - constraints = [Constraint(check="col > 0", name="chk")] + table_name = "blog_posts" + app_label = "blog" + database = "default" + ordering = ["-views"] + unique_together = [("title", "views")] + index_together = [("title", "views")] + indexes = [Index(fields=["title"], name="idx_post_title")] + constraints = [Constraint(check="views >= 0", name="chk_views_positive")] + abstract = False + managed = True +``` + +Model behavior: + +| API | Purpose | +|---|---| +| `Model(**kwargs)` | Create an unsaved instance with field defaults applied | +| `obj.pk` | Return the primary-key value | +| `await obj.full_clean()` | Run field validators and `clean()` | +| `await obj.save(validate=True, update_fields=None, using=None)` | Insert or update the row | +| `await obj.delete()` | Delete the row and clear its primary key | +| `await obj.refresh_from_db(fields=None)` | Reload fields from the database | +| `Model.DoesNotExist` | Per-model exception raised by `get()` | +| `Model.MultipleObjectsReturned` | Per-model exception raised by `get()` | + +Lifecycle hooks can be overridden: + +```python +class Post(Model): + title = CharField(max_length=200) + + async def clean(self): + if not self.title.strip(): + raise ValidationError({"title": ["Title cannot be empty"]}) + + async def before_save(self, created: bool): ... + async def after_save(self, created: bool): ... + async def before_delete(self): ... + async def after_delete(self): ... +``` + +## Managers + +Every concrete model receives `objects = Manager()` unless explicitly +overridden. + +```python +Post.objects.all() +Post.objects.filter(active=True) +Post.objects.exclude(status="draft") +Post.objects.order_by("-created_at") +Post.objects.using("replica") +Post.objects.cache(ttl=60) +Post.objects.stream(chunk_size=100) + +await Post.objects.create(title="Hello") +await Post.objects.get(pk=1) +await Post.objects.first() +await Post.objects.last() +await Post.objects.exists() +await Post.objects.count() +await Post.objects.aggregate(total=Count("id")) +await Post.objects.get_or_create(slug="hello", defaults={"title": "Hello"}) +await Post.objects.update_or_create(slug="hello", defaults={"title": "New"}) +await Post.objects.bulk_create([Post(title="A"), Post(title="B")]) +await Post.objects.bulk_update(posts, fields=["title"]) +await Post.objects.bulk_delete(Post.objects.filter(active=False)) ``` +## QuerySet + +`QuerySet` is lazy, async, chainable, and immutable. SQL is executed only when +the queryset is awaited or when an async terminal method is called. + +### Building Queries + +```python +qs = ( + Post.objects + .filter(active=True, views__gte=100) + .exclude(title__istartswith="draft") + .order_by("-views", "title") + .limit(20) + .offset(40) +) +posts = await qs +``` + +| Method | Purpose | +|---|---| +| `.filter(*q, **kwargs)` | Add AND-ed filters and `Q` expressions | +| `.exclude(*q, **kwargs)` | Add negated filters | +| `.all()` | Clone the queryset | +| `.order_by(*fields)` | Add ordering; `"-field"` means descending | +| `.limit(n)` | Add `LIMIT` | +| `.offset(n)` | Add `OFFSET` | +| `.distinct()` | Add `DISTINCT` | +| `.values(*fields)` | Select fields and enable grouping for annotations | +| `.annotate(**aggs)` | Add aggregate expressions per row | +| `.join(table, on, alias=None, kind="INNER")` | Add an explicit SQL join | +| `.using(alias)` | Force a database alias | +| `.schema(schema)` | Target a PostgreSQL schema | +| `.cache(ttl=None, key=None)` | Cache the first evaluated result | +| `.stream(chunk_size=100, keyset=None, as_dict=False)` | Iterate in chunks | + +Slicing is translated to `LIMIT`/`OFFSET`: + +```python +first_ten = await Post.objects.order_by("id")[:10] +page_two = await Post.objects.order_by("id")[10:20] +third = await Post.objects.order_by("id")[2] +``` + +Negative indexes and step slicing are not supported. + +`select_related()` exists on the Python queryset but is currently a no-op in +the implementation. Use explicit `.join()` when you need an eager SQL join. + +### Terminal Methods + +```python +await qs.count() +await qs.exists() +await qs.first() +await qs.get(pk=1) +await qs.update(views=100) +await qs.delete() +await qs.bulk_delete() +await qs.in_bulk([1, 2, 3]) +``` + +| Method | Return | +|---|---| +| `await qs` | `list[Model]` | +| `await qs.count()` | `int` | +| `await qs.exists()` | `bool` | +| `await qs.first()` | `Model | None` | +| `await qs.get(**filters)` | Exactly one model or per-model exception | +| `await qs.update(**fields)` | Number of updated rows | +| `await qs.delete()` | Number of deleted rows | +| `await qs.in_bulk(ids, field_name="pk")` | `dict[value, Model]` | + +Debug SQL: + +```python +print(Post.objects.filter(active=True).query) +``` + +## Q Objects + +```python +from ryx import Q + +Post.objects.filter(Q(active=True) | Q(views__gte=1000)) +Post.objects.filter(Q(active=True) & ~Q(status="draft")) +Post.objects.filter((Q(active=True) & Q(views__gte=100)) | Q(featured=True)) +``` + +Multiple kwargs inside one `Q()` are AND-ed. `Q` objects can be combined with +regular filter kwargs. + +## Lookups and Transforms + +Common lookups: + +```python +exact, gt, gte, lt, lte, +contains, icontains, +startswith, istartswith, +endswith, iendswith, +isnull, in, range +``` + +Date/time transforms: + +```python +date, year, month, day, hour, minute, second, week, +dow, quarter, time, iso_week, iso_dow +``` + +JSON transforms/lookups: + +```python +key, key_text, json, +has_key, has_any, has_all, contains, contained_by +``` + +Examples: + +```python +await Post.objects.filter(title__icontains="ryx") +await Post.objects.filter(created_at__year__gte=2025) +await Event.objects.filter(payload__key_text__icontains="error") +``` + +Runtime helpers: + +```python +ryx.available_lookups() +ryx.list_lookups() +ryx.available_transforms() +ryx.register_lookup("ne", "{col} <> ?") +``` + +Decorator form: + +```python +@ryx.lookup("ne") +def not_equal(): + """{col} <> ?""" +``` + +## Aggregates + +```python +from ryx import Count, Sum, Avg, Min, Max, RawAgg + +await Post.objects.aggregate( + total=Count("id"), + total_views=Sum("views"), + avg_views=Avg("views"), + min_views=Min("views"), + max_views=Max("views"), +) + +rows = await ( + Post.objects + .values("author_id") + .annotate(post_count=Count("id"), views=Sum("views")) +) +``` + +`Count("field", distinct=True)` produces a distinct count. `RawAgg(sql, +alias)` is available for backend-specific expressions. + ## Fields ```python @@ -45,69 +311,112 @@ from ryx import ( ) ``` -## QuerySet +Common field options: ```python -qs.filter(**kwargs) -qs.exclude(**kwargs) -qs.all() -qs.get(**kwargs) -qs.first() -qs.last() -qs.exists() -qs.count() -qs.order_by(*fields) -qs.limit(n) -qs.offset(n) -qs.distinct() -qs.values(*fields) -qs.annotate(**kwargs) -qs.aggregate(**kwargs) -qs.join(table, condition, alias=None, kind="INNER") -qs.update(**kwargs) -qs.delete() -qs.cache() -qs.stream(page_size=500) -qs.using(db_alias) -qs.in_bulk(id_list) +CharField( + max_length=200, + null=False, + blank=False, + default="", + primary_key=False, + unique=False, + db_index=False, + choices=["draft", "published"], + validators=[], + editable=True, + help_text="", + verbose_name="", + db_column="db_column_name", + unique_for_date=None, + unique_for_month=None, + unique_for_year=None, +) ``` -## Q Objects +See [Field Reference](/reference/field-reference) for the full field matrix. + +## Relationships ```python -from ryx import Q +class Author(Model): + name = CharField(max_length=100) + +class Post(Model): + author = ForeignKey(Author, on_delete="CASCADE", related_name="posts") + +class Profile(Model): + author = OneToOneField(Author, on_delete="CASCADE") + +class Tag(Model): + name = CharField(max_length=50) + +class TaggedPost(Model): + post = ForeignKey(Post, on_delete="CASCADE") + tag = ForeignKey(Tag, on_delete="CASCADE") -Q(field=value) -Q(field__lookup=value) -Q(a=True) | Q(b=True) # OR -Q(a=True) & Q(b=True) # AND -~Q(a=True) # NOT +class PostTag(Model): + tags = ManyToManyField(Tag, through=TaggedPost) ``` -## Aggregates +Relationship helpers include forward descriptors, reverse FK managers, and +many-to-many managers. Reverse FK managers expose queryset-like helpers such as +`all()`, `filter()`, `exclude()`, `order_by()`, `count()`, `exists()`, +`first()`, `get()`, `create()`, `add()`, `remove()`, and `delete()`. + +## Transactions ```python -from ryx import Count, Sum, Avg, Min, Max, RawAgg +from ryx import transaction, get_active_transaction + +async with transaction("default") as tx: + await Account.objects.get(pk=1) + tx.savepoint("before_transfer") + tx.rollback_to("before_transfer") + tx.release_savepoint("before_transfer") -Count("field", distinct=True) -Sum("field") -Avg("field") -Min("field") -Max("field") -RawAgg("SQL expression") +current = get_active_transaction() ``` -## Transactions +The transaction context commits on successful exit and rolls back on exception. +Nested transactions are implemented with savepoints by the Rust backend. + +## Bulk Operations and Streaming ```python -import ryx +from ryx import bulk_create, bulk_update, bulk_delete, stream + +await bulk_create([Post(title="A"), Post(title="B")], batch_size=500) +await bulk_update(posts, fields=["title"], batch_size=500) +await bulk_delete(Post.objects.filter(active=False)) + +async for post in Post.objects.filter(active=True).stream(chunk_size=100): + ... + +async for row in Post.objects.stream(chunk_size=500, keyset="id", as_dict=True): + ... +``` + +`keyset` streaming uses cursor-style pagination and is preferred for large +tables when the keyset column is indexed. + +## Validation -async with ryx.transaction() as tx: - tx.savepoint("name") - tx.rollback_to("name") - tx.release_savepoint("name") +```python +from ryx import ( + ValidationError, Validator, FunctionValidator, + NotNullValidator, NotBlankValidator, + MaxLengthValidator, MinLengthValidator, + MinValueValidator, MaxValueValidator, + RangeValidator, RegexValidator, + EmailValidator, URLValidator, ChoicesValidator, +) +``` + +Validation runs in `save()` by default. Pass `validate=False` to skip it: -ryx.get_active_transaction() # → TransactionHandle | None +```python +await post.save(validate=False) ``` ## Signals @@ -121,118 +430,161 @@ from ryx import ( pre_bulk_delete, post_bulk_delete, ) +@receiver(post_save, sender=Post) +async def on_post_saved(sender, instance, created, **kwargs): + ... +``` + +Signal methods: + +```python signal.connect(handler, sender=None, weak=True) signal.disconnect(handler, sender=None) await signal.send(sender, **kwargs) - -@receiver(signal, sender=Model) -async def handler(sender, **kwargs): ... ``` -## Validation +## Caching ```python -from ryx import ValidationError -from ryx.validators import ( - MaxLengthValidator, MinLengthValidator, - MaxValueValidator, MinValueValidator, - RangeValidator, RegexValidator, - EmailValidator, URLValidator, - NotBlankValidator, ChoicesValidator, - FunctionValidator, NotNullValidator, - UniqueValueValidator, - run_full_validation, -) -``` +from ryx import MemoryCache, configure_cache, get_cache +from ryx import invalidate, invalidate_model, invalidate_all -## Bulk Operations +configure_cache(MemoryCache(max_size=1000, ttl=300)) -```python -from ryx.bulk import bulk_create, bulk_update, bulk_delete, stream +posts = await Post.objects.filter(active=True).cache(ttl=60) +named = await Post.objects.all().cache(key="all_posts", ttl=300) -await bulk_create(instances, batch_size=100) -await bulk_update(instances, fields=["field"]) -await bulk_delete(queryset) -async for batch in stream(queryset, page_size=500): ... +await invalidate("all_posts") +await invalidate_model(Post) +await invalidate_all() +cache = get_cache() ``` -## Relations +`MemoryCache` implements `get`, `set`, `delete`, `delete_many`, `clear`, and +`keys`. It is process-local and TTL-aware. + +## Raw SQL and Parameterized Execution ```python -from ryx.relations import apply_select_related, apply_prefetch_related +from ryx.executor_helpers import raw_fetch, raw_execute +from ryx.pool_ext import fetch_with_params, execute_with_params -await apply_select_related(queryset, fields=["author"]) -await apply_prefetch_related(queryset, fields=["tags"]) +rows = await raw_fetch("SELECT * FROM posts WHERE active = true", alias="default") +await raw_execute("CREATE INDEX IF NOT EXISTS idx_posts_active ON posts(active)") + +rows = await fetch_with_params("SELECT * FROM posts WHERE id = ?", [1]) +affected = await execute_with_params("UPDATE posts SET views = ? WHERE id = ?", [10, 1]) ``` -## Caching +Use parameterized helpers for user input. Raw SQL helpers execute the SQL string +directly. + +## Database Routing ```python -from ryx import configure_cache, get_cache +from ryx.router import BaseRouter, set_router -configure_cache(ttl=300, max_size=1000) -cache = get_cache() +class Router(BaseRouter): + def db_for_read(self, model, **hints): + return "replica" + + def db_for_write(self, model, **hints): + return getattr(model._meta, "database", None) or "default" + + def allow_migrate(self, db, app_label, model_name): + return True + +set_router(Router()) ``` -## Custom Lookups +Read alias resolution order: + +1. QuerySet `.using(alias)` +2. `Router.db_for_read(model)` +3. `Model.Meta.database` +4. `"default"` + +Write alias resolution order: + +1. Explicit `save(using=alias)` or QuerySet `.using(alias)` where applicable +2. `Router.db_for_write(model)` +3. `Model.Meta.database` +4. `"default"` + +## Migrations ```python -import ryx +from ryx.migrations import MigrationRunner +from ryx.migrations.autodetect import Autodetector +from ryx.migrations.ddl import DDLGenerator, generate_schema_ddl, detect_backend -ryx.register_lookup("name", "{col} OPERATOR ?") -ryx.available_lookups() # → list[str] +runner = MigrationRunner([Author, Post], migrations_dir="migrations") +changes = await runner.migrate() -@ryx.lookup("name") -def my_lookup(field, value): - """{col} OPERATOR ?""" +detector = Autodetector([Author, Post], migrations_dir="migrations") +operations = detector.detect() +path = detector.write_migration(operations) ``` -## Sync/Async +Operation classes: ```python -from ryx import run_sync, sync_to_async, async_to_sync, run_async - -run_sync(awaitable) -sync_to_async(func) -async_to_async(func) -async_to_sync(awaitable) -await run_async(func, *args) +CreateTable(table, columns, schema="", model=None) +AddField(table, column, schema="", model=None) +AlterField(table, new_col, old_col=None, schema="", model=None) +CreateIndex(table, name, fields, unique=False, schema="", model=None) +RunSQL(sql, reverse_sql="") ``` -## Raw SQL +DDL helpers support PostgreSQL, MySQL, and SQLite and include schema-aware SQL +generation for PostgreSQL. + +## Sync and Async Bridge ```python -from ryx.executor_helpers import raw_fetch, raw_execute -from ryx.pool_ext import fetch_with_params, execute_with_params +from ryx import run_sync, run_async, sync_to_async, async_to_sync -await raw_fetch("SELECT ...", [params]) -await raw_execute("CREATE ...") -await fetch_with_params("SELECT ... WHERE x = ?", [value]) -await execute_with_params("INSERT ...", [values]) +posts = run_sync(Post.objects.filter(active=True)) + +async_fn = sync_to_async(blocking_function) +result = await async_fn() + +wrapped = async_to_sync(async_function) +value = wrapped() + +result = await run_async(blocking_function, arg1, key="value") ``` -## Migrations +Prefer native `await` inside async applications. Use the bridge helpers for +scripts, WSGI handlers, and synchronous integrations. + +## Exceptions ```python -from ryx.migrations import MigrationRunner, DDLGenerator, generate_schema_ddl +from ryx import ( + RyxError, + DatabaseError, + DoesNotExist, + MultipleObjectsReturned, + PoolNotInitialized, + ValidationError, +) +``` -runner = MigrationRunner([Model1, Model2]) -await runner.migrate() -await runner.migrate(dry_run=True) +`DoesNotExist` and `MultipleObjectsReturned` also exist as per-model subclasses: -stmts = generate_schema_ddl([Model1], backend="postgres") +```python +try: + await Post.objects.get(slug="missing") +except Post.DoesNotExist: + ... ``` -## CLI +## CLI Entrypoints ```bash -python -m ryx migrate --url ... --models ... -python -m ryx makemigrations --models ... --dir ... -python -m ryx showmigrations --url ... --dir ... -python -m ryx sqlmigrate --dir ... -python -m ryx flush --models ... --url ... --yes -python -m ryx shell --url ... --models ... -python -m ryx dbshell --url ... -python -m ryx inspectdb --url ... [--table ...] -python -m ryx version +python -m ryx --url sqlite:///app.db migrate --models myapp.models +ryx --url sqlite:///app.db migrate --models myapp.models ``` + +See [CLI](/advanced/cli) for command options and config file formats. diff --git a/docs/doc/reference/field-reference.mdx b/docs/doc/reference/field-reference.mdx index 6e86c36..9784ac8 100644 --- a/docs/doc/reference/field-reference.mdx +++ b/docs/doc/reference/field-reference.mdx @@ -4,78 +4,215 @@ sidebar_position: 3 # Field Reference -Complete reference of all field types, their SQL types, and Python types. +Ryx fields are descriptors. A field stores model metadata, validates Python +values, converts values to database values, and exposes SQL type information to +the migration system. + +## Common Field Options + +All field classes inherit these options from `Field`: + +| Option | Default | Purpose | +|---|---:|---| +| `null` | `False` | Allows SQL `NULL` | +| `blank` | `False` | Allows empty values during validation | +| `default` | none | Static value or callable default | +| `primary_key` | `False` | Marks the model primary key | +| `unique` | `False` | Adds uniqueness validation/constraint metadata | +| `db_index` | `False` | Marks the field for index creation | +| `choices` | `None` | Restricts values with `ChoicesValidator` | +| `validators` | `[]` | Extra validator instances | +| `editable` | `True` | Excluded from `save()` when `False`, except auto timestamp fields | +| `help_text` | `""` | Human-readable description | +| `verbose_name` | `""` | Human-readable label | +| `db_column` | field name | Overrides the SQL column name | +| `unique_for_date` | `None` | Declares date-scoped uniqueness metadata | +| `unique_for_month` | `None` | Declares month-scoped uniqueness metadata | +| `unique_for_year` | `None` | Declares year-scoped uniqueness metadata | + +```python +title = CharField( + max_length=200, + null=False, + blank=False, + default="Untitled", + unique=True, + db_index=True, + choices=["draft", "published"], + db_column="post_title", +) +``` + +If no primary key is declared on a concrete model, Ryx injects: + +```python +id = AutoField(primary_key=True, editable=False) +``` ## Integer Fields -| Field | SQL Type | Python Type | Extra Kwargs | +| Field | SQL Type | Python Type | Extra Options | |---|---|---|---| -| `AutoField` | `SERIAL` | `int` | — | -| `BigAutoField` | `BIGSERIAL` | `int` | — | -| `SmallAutoField` | `SMALLSERIAL` | `int` | — | +| `AutoField` | `SERIAL` | `int` | Usually primary key | +| `BigAutoField` | `BIGSERIAL` | `int` | Large auto primary key | +| `SmallAutoField` | `SMALLSERIAL` | `int` | Small auto primary key | | `IntField` | `INTEGER` | `int` | `min_value`, `max_value` | | `SmallIntField` | `SMALLINT` | `int` | `min_value`, `max_value` | | `BigIntField` | `BIGINT` | `int` | `min_value`, `max_value` | -| `PositiveIntField` | `INTEGER` | `int` | implicit `min_value=0` | +| `PositiveIntField` | `INTEGER` | `int` | Adds `min_value=0` behavior | + +```python +class Product(Model): + stock = PositiveIntField(default=0) + rating = SmallIntField(min_value=1, max_value=5) +``` + +## Numeric Fields + +| Field | SQL Type | Python Type | Extra Options | +|---|---|---|---| +| `FloatField` | `DOUBLE PRECISION` | `float` | `min_value`, `max_value` | +| `DecimalField` | `NUMERIC(max_digits, decimal_places)` | `Decimal` | `max_digits`, `decimal_places` | + +```python +price = DecimalField(max_digits=10, decimal_places=2) +score = FloatField(min_value=0.0, max_value=1.0) +``` + +## Boolean Fields + +| Field | SQL Type | Python Type | Notes | +|---|---|---|---| +| `BooleanField` | `BOOLEAN` | `bool` | Standard boolean | +| `NullBooleanField` | `BOOLEAN` | `bool | None` | Sets `null=True` | ## Text Fields -| Field | SQL Type | Python Type | Extra Kwargs | +| Field | SQL Type | Python Type | Extra Options | |---|---|---|---| -| `CharField` | `VARCHAR(n)` | `str` | `max_length`, `min_length`, `strip` | +| `CharField` | `VARCHAR(max_length)` | `str` | `max_length`, `min_length`, `strip` | | `TextField` | `TEXT` | `str` | `min_length` | -| `EmailField` | `VARCHAR(254)` | `str` | auto email validation | -| `SlugField` | `VARCHAR(50)` | `str` | auto slug validation | -| `URLField` | `VARCHAR(200)` | `str` | auto URL validation | -| `IPAddressField` | `VARCHAR(15)` | `str` | auto IPv4 validation | +| `SlugField` | `VARCHAR(50)` | `str` | Slug-style validation | +| `EmailField` | `VARCHAR(254)` | `str` | Email validation | +| `URLField` | `VARCHAR(200)` | `str` | URL validation | +| `IPAddressField` | `VARCHAR(15)` | `str` | IPv4 validation | + +```python +slug = SlugField(unique=True) +email = EmailField(unique=True) +bio = TextField(blank=True) +``` -## Date & Time Fields +## Date and Time Fields -| Field | SQL Type | Python Type | Extra Kwargs | +| Field | SQL Type | Python Type | Extra Options | |---|---|---|---| | `DateField` | `DATE` | `date` | `auto_now`, `auto_now_add` | | `DateTimeField` | `TIMESTAMP` | `datetime` | `auto_now`, `auto_now_add` | | `TimeField` | `TIME` | `time` | — | -| `DurationField` | `BIGINT` | `timedelta` | stored as microseconds | +| `DurationField` | `BIGINT` | `timedelta` | Stored as microseconds | + +```python +created_at = DateTimeField(auto_now_add=True) +updated_at = DateTimeField(auto_now=True) +publish_date = DateField(null=True) +``` -## Special Fields +`auto_now_add` updates only on insert. `auto_now` updates on every save. -| Field | SQL Type | Python Type | Extra Kwargs | +## Structured and Binary Fields + +| Field | SQL Type | Python Type | Extra Options | |---|---|---|---| -| `BooleanField` | `BOOLEAN` | `bool` | — | -| `NullBooleanField` | `BOOLEAN` | `bool \| None` | implicit `null=True` | -| `FloatField` | `DOUBLE PRECISION` | `float` | `min_value`, `max_value` | -| `DecimalField` | `NUMERIC(p,s)` | `Decimal` | `max_digits`, `decimal_places` | -| `UUIDField` | `UUID` | `UUID` | `auto_create` | -| `JSONField` | `JSONB` | `dict \| list` | — | +| `UUIDField` | `UUID` | `uuid.UUID` | `auto_create` | +| `JSONField` | `JSONB` | `dict | list` | JSON lookups/transforms | +| `ArrayField` | `[]` | `list` | `base_field` | | `BinaryField` | `BYTEA` | `bytes` | — | -| `ArrayField` | `T[]` | `list` | `base_field` | + +```python +uid = UUIDField(auto_create=True, unique=True) +metadata = JSONField(default=dict) +tags = ArrayField(CharField(max_length=40), default=list) +payload = BinaryField(null=True) +``` + +JSON lookups include `has_key`, `has_any`, `has_all`, `contains`, and +`contained_by`. JSON transforms include `key`, `key_text`, and `json`. ## Relationship Fields -| Field | SQL Type | Python Type | Extra Kwargs | +| Field | Database Representation | Python Use | Extra Options | |---|---|---|---| -| `ForeignKey` | `INTEGER` | `int` | `on_delete`, `related_name` | -| `OneToOneField` | `INTEGER UNIQUE` | `int` | same as FK | -| `ManyToManyField` | *(join table)* | — | `through` | +| `ForeignKey` | FK id column | Forward descriptor + reverse manager | `to`, `on_delete`, `related_name` | +| `OneToOneField` | Unique FK id column | One-to-one descriptor | Same as `ForeignKey` | +| `ManyToManyField` | Through table | Many-to-many manager | `to`, `through`, `related_name` | -## Common Field Options +```python +class Author(Model): + name = CharField(max_length=100) + +class Post(Model): + author = ForeignKey(Author, on_delete="CASCADE", related_name="posts") + +class Profile(Model): + author = OneToOneField(Author, on_delete="CASCADE") +``` + +Supported `on_delete` values are interpreted by the field and migration layer. +Use the exact values demonstrated in examples and tests, such as `"CASCADE"`. + +## Validation + +Field validation combines: + +1. `NotNullValidator` for non-null, non-primary-key fields +2. `ChoicesValidator` when `choices` is provided +3. `UniqueValueValidator` when `unique=True` +4. Field-specific validators such as length, range, email, URL, or regex +5. Explicit validators passed in `validators=[...]` ```python -CharField( - max_length=200, # Required for CharField - min_length=5, # Optional minimum - null=True, # Allow NULL in DB - blank=True, # Allow empty in validation - default="", # Default value or callable - unique=True, # UNIQUE constraint - db_index=True, # CREATE INDEX - choices=["a", "b"], # Restrict values - validators=[...], # Extra validators - editable=True, # Include in save() - help_text="...", # Documentation - verbose_name="...", # Human-readable label - db_column="col", # Override column name - primary_key=False, # Make this the PK +from ryx import CharField, RegexValidator + +username = CharField( + max_length=40, + validators=[RegexValidator(r"^[a-z0-9_]+$")], ) ``` + +## Lookup Validation + +Each field class declares supported lookups and transforms. Ryx validates lookup +chains before sending a query to Rust: + +```python +await Post.objects.filter(title__icontains="ryx") +await Post.objects.filter(created_at__year__gte=2025) +await Post.objects.filter(metadata__key_text__icontains="error") +``` + +Unknown lookup suffixes fall back to exact matching during parsing, so prefer +`ryx.available_lookups()` and this reference when authoring advanced filters. + +## Migration Metadata + +Fields expose `db_type()` and `deconstruct()` data used by: + +- `project_state_from_models()` +- `Autodetector` +- `DDLGenerator` +- `MigrationRunner` + +For unmanaged legacy tables, set: + +```python +class LegacyUser(Model): + name = CharField(max_length=100) + + class Meta: + table_name = "legacy_users" + managed = False +``` + +`managed=False` keeps the model queryable while preventing migration generation +from creating or altering its table. diff --git a/docs/doc/rust/core-concepts/migrations.mdx b/docs/doc/rust/core-concepts/migrations.mdx index 4687859..9c3455d 100644 --- a/docs/doc/rust/core-concepts/migrations.mdx +++ b/docs/doc/rust/core-concepts/migrations.mdx @@ -4,51 +4,259 @@ sidebar_position: 6 # Migrations -Ryx can detect your models and create database tables automatically. +Ryx provides a complete migration system for Rust: live auto-diff, file-based YAML migrations with per-database routing, and a CLI for every workflow. -## Basic Usage +## Two Approaches + +### Live Migration (No Files) + +Best for prototyping and simple projects. Introspects the DB, diffs against models, and applies DDL directly: ```rust use ryx_rs::migration::MigrationRunner; MigrationRunner::new() - .model::() + .live(true) .model::() + .model::() .run().await?; ``` -This introspects your database, diffs against your model definitions, and generates the necessary DDL (CREATE TABLE, ALTER COLUMN, etc.). +Set `.live(true)` to skip file discovery and apply changes directly. Omitting it enters the file-based pipeline (see below). + +### File-Based YAML Migrations + +Best for production and team projects. Migration files are plain YAML, discovered recursively, and tracked per-database alias: + +```bash +# Generate migration files (coming soon — use Python for now) +ryx makemigrations --dir migrations/ + +# Apply pending migrations +ryx migrate --dir migrations/ + +# Preview SQL +ryx sqlmigrate 0001_initial --dir migrations/ --backends postgres,mysql,sqlite + +# Check status +ryx showmigrations --dir migrations/ +``` + +## CLI Reference + +```bash +ryx migrate [--dir DIR] [--database ALIAS] [--dry-run] [--no-interactive] +ryx makemigrations --dir DIR [--alias ALIAS] [--check] +ryx showmigrations [--dir DIR] [--unapplied] +ryx sqlmigrate NAME [--dir DIR] [--backends postgres,mysql,sqlite] +``` + +The CLI uses ANSI colours when the terminal supports it. Set `NO_COLOR=1` to disable, or pipe output. ## How It Works ``` -1. Introspect live DB schema → SchemaState (current) -2. Build target from models → SchemaState (desired) -3. Diff the two states → List of changes -4. Generate DDL → CREATE / ALTER statements -5. Execute → Apply to database +1. Discover all [0-9]*.yaml files recursively → sorted by numeric prefix +2. For each DB alias, filter operations → via operation.database() +3. Convert operations to backend-aware DDL → DDLGenerator +4. Execute statements → per-backend SQL +5. Track applied state → alias|stem in ryx_migrations +``` + +When no migration files exist, `ryx migrate` offers an **interactive menu**: + +``` +[ryx] No migration files exist for database 'default' + 2 model(s) are not yet tracked. + + [L]ive DDL — apply changes directly (development only) + [A]uto-generate migration files, then migrate + [M]anual — run 'ryx makemigrations' first + [S]kip this database for now + +[ryx] Choice (L/A/M/S) [S]: ``` -## What Migrations Handle +Use `--no-interactive` to skip the prompt in scripts or CI/CD (prints a hint and exits). -- Creating and dropping tables -- Adding, altering, and dropping columns -- Creating and dropping indexes -- Adding constraints -- Creating ManyToMany join tables +## Multi-Database Routing -## Tracking +Each model declares its database alias with `#[database("alias")]`: + +```rust +#[model] +#[database("blog")] +#[table("posts")] +struct Post { + #[field(pk)] id: i64, + title: String, +} + +#[model] +#[database("shop")] +#[table("products")] +struct Product { + #[field(pk)] id: i64, + name: String, +} +``` + +The migration runner filters operations per alias automatically. The tracking table stores `alias|stem` keys: + +| Column | Type | Example | +|---|---|---| +| id | INTEGER | 1 | +| name | TEXT | `default\|0001_initial` | +| applied_at | TIMESTAMP | 2025-06-03 12:00:00 | + +Legacy entries (bare stems without `|`) are still recognised for backward compatibility. + +## YAML File Format + +Migration files are human-readable YAML, one file per migration: + +```yaml +# migrations/0001_initial.yaml +dependencies: [] +operations: + - type: CreateTable + table_name: authors + model_name: myapp::Author + database: default + columns: + - { name: id, db_type: INTEGER, pk: true, unique: true } + - { name: name, db_type: VARCHAR(100), nullable: false } + + - type: CreateTable + table_name: posts + model_name: myapp::Post + database: blog + columns: + - { name: id, db_type: INTEGER, pk: true, unique: true } + - { name: title, db_type: VARCHAR(200), nullable: false } + - { name: author_id, db_type: INTEGER, nullable: false } +``` + +Supported operation types: `CreateTable`, `AddField`, `RemoveField`, `AlterField`, `CreateIndex`, `DeleteIndex`, `RunSQL`. + +## Programmatic API + +### MigrationRunner (Builder) + +```rust +use ryx_rs::migration::MigrationRunner; + +// File-based (default) +MigrationRunner::new() + .model::() + .migrations_dir("migrations/") + .db("blog") + .dry_run(true) + .run().await?; + +// Live mode — skip file discovery +MigrationRunner::new() + .live(true) + .model::() + .run().await?; + +// Live mode with schema (PostgreSQL multi-schema) +MigrationRunner::new() + .live(true) + .model::() + .schema("tenant1") + .run().await?; + +// Preview only +let sql: Vec = MigrationRunner::new() + .model::() + .plan().await?; +``` + +### FileRunner (Programmatic) + +```rust +use ryx_rs::migration::runner::FileRunner; + +let runner = FileRunner::new() + .model::() + .migrations_dir("migrations/") + .no_interactive(true); + +let statements: Vec = runner.run().await?; +``` + +### DDLGenerator + +```rust +use ryx_rs::migration::ddl::DDLGenerator; +use ryx_query::Backend; + +let ddl = DDLGenerator::new(Backend::PostgreSQL); + +let sql = ddl.create_table(&table_state); +let sql = ddl.drop_table("posts"); +let sql = ddl.add_column("posts", &col_state); +let sql = ddl.create_index("posts", "idx_title", &["title".into()], false); +let sql = ddl.add_foreign_key("posts", "fk_author", "author_id", "authors", "id"); + +// Bulk generation from SchemaChange list +let all_sql = ddl.generate(&changes); +``` + +### Autodetector (Generate YAML files) + +```rust +use ryx_rs::migration::autodetect::{Autodetector, ModelEntry}; + +let entries = vec![ + ModelEntry::from_model::(), + ModelEntry::from_model::(), +]; + +let detector = Autodetector::new(entries, "migrations/"); + +// Detect changes → write migration file +if let Some(path) = detector.run()? { + println!("Created {}", path.display()); +} + +// Or step-by-step +let ops = detector.detect(); +if !ops.is_empty() { + detector.write_migration(&ops)?; +} +``` + +## File Discovery + +Migration files are discovered **recursively** under the migrations directory. Files matching `[0-9]*.yaml` or `[0-9]*.yml` are found and sorted globally by numeric prefix: + +``` +migrations/ +├── 0001_initial.yaml +├── 0002_add_views.yaml +├── blog/ +│ ├── 0001_blog_tables.yaml +│ └── 0002_add_tags.yaml +└── shop/ + └── 0001_shop_tables.yaml +``` -Ryx creates a `ryx_migrations` table to track applied state. +Files in subdirectories are found automatically — no per-alias configuration needed. ## Backend Differences | Feature | PostgreSQL | MySQL | SQLite | |---|---|---|---| -| `ALTER COLUMN` | Yes | Yes | No (recreate) | +| `ALTER COLUMN` | `ALTER COLUMN ... TYPE` | `MODIFY COLUMN` | Manual rebuild required | +| `IF NOT EXISTS` / `IF EXISTS` | Yes | Yes | Yes | +| `DROP COLUMN` | Yes | Yes | Not supported (comment) | | Native UUID | Yes | No | No | -| `SERIAL` | Yes | No | No | +| `SERIAL` | `BIGSERIAL` | `INT AUTO_INCREMENT` | `INTEGER PRIMARY KEY AUTOINCREMENT` | +| **Database schemas** | **`"schema"."table"`** | No | No | ## Next Steps -- **[Querying](/rust/querying/filtering)** — Start querying your data +- **[Models](./models)** — Define models with `#[model]` +- **[Querying](../querying/filtering)** — Start querying your data diff --git a/docs/doc/rust/core-concepts/queryset.mdx b/docs/doc/rust/core-concepts/queryset.mdx index 2d0e064..f5e2e96 100644 --- a/docs/doc/rust/core-concepts/queryset.mdx +++ b/docs/doc/rust/core-concepts/queryset.mdx @@ -4,7 +4,21 @@ sidebar_position: 5 # QuerySet -`QuerySet` is the lazy, chainable query builder. Every model's `objects()` manager returns one. +`QuerySet` is the lazy, chainable query builder. The canonical entry point is +`ObjectsManager::::new()`. + +Some examples use a short `Post::objects()` helper. That helper is not required +by the macro; define it yourself if you want the shorter style: + +```rust +use ryx_rs::ObjectsManager; + +impl Post { + fn objects() -> ObjectsManager { + ObjectsManager::new() + } +} +``` ## Entry Points diff --git a/docs/doc/rust/getting-started/quick-start.mdx b/docs/doc/rust/getting-started/quick-start.mdx index e68d222..93ec2d8 100644 --- a/docs/doc/rust/getting-started/quick-start.mdx +++ b/docs/doc/rust/getting-started/quick-start.mdx @@ -10,6 +10,7 @@ Let's go from zero to a working query in 5 minutes. ```rust use ryx_rs::model; +use ryx_rs::ObjectsManager; #[model] #[table("posts")] @@ -35,6 +36,7 @@ use ryx_rs::migration::MigrationRunner; async fn run() -> ryx_rs::RyxResult<()> { ryx_rs::init().await?; MigrationRunner::new() + .live(true) // apply directly — no files needed .model::() .run().await?; Ok(()) @@ -43,10 +45,22 @@ async fn run() -> ryx_rs::RyxResult<()> { `init()` reads a `ryx.toml` or uses the `DATABASE_URL` env var. +:::tip +For production, use **file-based YAML migrations** with the CLI: + +```bash +ryx makemigrations --dir migrations/ +ryx migrate --dir migrations/ +ryx showmigrations +``` + +See [Migrations](../core-concepts/migrations) for the full guide. +::: + ## 3. Create Records ```rust -let post = Post::objects().create() +let post = ObjectsManager::::new().create() .set("title", "Hello Ryx") .set("slug", "hello-ryx") .set("views", 42i64) @@ -58,24 +72,24 @@ let post = Post::objects().create() ```rust // All active posts, newest first -let posts: Vec = Post::objects() +let posts: Vec = ObjectsManager::::new() .filter("active", true) .order_by("-views") .all().await?; // First match -let first: Option = Post::objects() +let first: Option = ObjectsManager::::new() .filter("active", true) .order_by("title") .first().await?; // Count -let count = Post::objects() +let count = ObjectsManager::::new() .filter("active", true) .count().await?; // Check existence -let exists = Post::objects() +let exists = ObjectsManager::::new() .filter("title__startswith", "Draft") .exists().await?; ``` @@ -84,13 +98,13 @@ let exists = Post::objects() ```rust // Bulk update -Post::objects() +ObjectsManager::::new() .filter("active", false) .update(vec![("active", true)]) .await?; // Bulk delete -Post::objects() +ObjectsManager::::new() .filter("views", 0i64) .delete().await?; ``` @@ -100,6 +114,7 @@ Post::objects() ```rust use ryx_rs::model; use ryx_rs::migration::MigrationRunner; +use ryx_rs::ObjectsManager; #[model] #[table("posts")] @@ -115,22 +130,23 @@ struct Post { #[tokio::main] async fn main() -> ryx_rs::RyxResult<()> { ryx_rs::init().await?; - MigrationRunner::new().model::().run().await?; + MigrationRunner::new().live(true).model::().run().await?; - Post::objects().create() + ObjectsManager::::new().create() .set("title", "Hello Ryx") .set("slug", "hello-ryx") .set("views", 100i64) .save().await?; - let posts: Vec = Post::objects() + let posts: Vec = ObjectsManager::::new() .filter("active", true) .order_by("-views") .all().await?; println!("Found {} posts", posts.len()); - let stats = Post::objects() + let stats = ObjectsManager::::new() + .all() .aggregate(&[ ryx_rs::agg::count("total", "id"), ryx_rs::agg::avg("avg", "views"), diff --git a/docs/doc/rust/index.mdx b/docs/doc/rust/index.mdx index 3178ab6..de4e3f7 100644 --- a/docs/doc/rust/index.mdx +++ b/docs/doc/rust/index.mdx @@ -8,7 +8,7 @@ slug: /rust Ryx is a Django-style ORM for Rust. Async-native, with zero Python dependencies. ```rust -use ryx_rs::model; +use ryx_rs::{model, ObjectsManager}; #[model] struct Post { @@ -19,12 +19,24 @@ struct Post { active: bool, } -let posts = Post::objects() +let posts = ObjectsManager::::new() .filter("active", true) .order_by("-views") .all().await?; ``` +Many older snippets use a convenience helper named `Post::objects()`. The macro +does not need that helper; the canonical entry point is +`ObjectsManager::::new()`. If you prefer the shorter style, define it once: + +```rust +impl Post { + fn objects() -> ObjectsManager { + ObjectsManager::new() + } +} +``` + ## Crates | Crate | Description | @@ -37,12 +49,15 @@ let posts = Post::objects() | Feature | Python (ryx) | Rust (ryx-rs) | |---|---|---| | Model definition | Class + Fields | `#[model]` struct | -| QuerySet | `Post.objects` | `Post::objects()` | +| QuerySet | `Post.objects` | `ObjectsManager::::new()` or a user-defined `Post::objects()` helper | | Filtering | `.filter(active=True)` | `.filter("active", true)` | | Q objects | `Q(active=True) \| Q(...)` | `Q::or(Q::new(...), ...)` | | Create | `.create(title="...")` | `.create().set("title", "...").save()` | | Migrations | `MigrationRunner([Post])` | `MigrationRunner::new().model::()` | | Relations | `ForeignKey(Author)` | `#[relation(model = "Author", ...)]` | +| CLI | `python -m ryx migrate` | `ryx migrate` | +| File-based migrations | `.py` files + Autodetector | `.yaml` files + Autodetector | +| Multi-DB routing | `Meta.database` + `model=` | `#[database("alias")]` | ## Next Steps diff --git a/docs/doc/rust/reference.mdx b/docs/doc/rust/reference.mdx new file mode 100644 index 0000000..21a5617 --- /dev/null +++ b/docs/doc/rust/reference.mdx @@ -0,0 +1,292 @@ +--- +sidebar_position: 99 +--- + +# Rust API Reference + +The Rust API lives in `ryx-rs` and reuses the shared crates: + +- `ryx-query` for query AST, lookups, and SQL compilation +- `ryx-backend` for sqlx-backed PostgreSQL, MySQL, and SQLite execution +- `ryx-common` for shared errors, pool config, and SQL values +- `ryx-macro` for model derive and attribute macros + +## Initialization + +```rust +use ryx_rs::{init, RyxResult}; + +#[tokio::main] +async fn main() -> RyxResult<()> { + init().await?; + Ok(()) +} +``` + +`init()` loads config files and environment variables, initializes tracing from +`RYX_LOG_LEVEL`, and initializes the global pool if configuration is present. + +## Model Macros + +```rust +use ryx_rs::model; + +#[model] +struct Post { + #[field(pk)] + id: i64, + title: String, + views: i64, + active: bool, +} +``` + +The public model traits are: + +| Trait | Purpose | +|---|---| +| `Model` | Table name, field names, primary key, field metadata, database alias | +| `FromRow` | Convert decoded backend rows into model instances | +| `Relationships` | Optional relationship metadata for `select_related()` | + +## Objects Manager + +```rust +use ryx_rs::ObjectsManager; + +let posts = ObjectsManager::::new().all().all().await?; +let active = ObjectsManager::::new().filter("active", true).all().await?; +let post = ObjectsManager::::new().all().get("id", 1).await?; + +let created = ObjectsManager::::new() + .create() + .set("title", "Hello") + .set("views", 0) + .set("active", true) + .save() + .await?; +``` + +`ObjectsManager::::new()` exposes: + +| Method | Purpose | +|---|---| +| `.all()` | Start a `QuerySet` | +| `.filter(field, value)` | Start a filtered queryset | +| `.exclude(field, value)` | Start an excluded queryset | +| `.get(field, value)` | Start a queryset filtered for `get()` | +| `.create()` | Start an `InsertBuilder` | + +## QuerySet + +```rust +let posts = ObjectsManager::::new() + .all() + .filter(("views__gte", 100)) + .exclude(("active", false)) + .order_by("-views") + .limit(20) + .offset(40) + .all() + .await?; +``` + +Query building: + +| Method | Purpose | +|---|---| +| `.using(alias)` | Force a database alias | +| `.schema(schema)` | Target a PostgreSQL schema | +| `.filter(arg)` | Add a field lookup or `Q` object | +| `.exclude(arg)` | Add a negated field lookup or negated `Q` | +| `.order_by(field)` | Add one ordering clause | +| `.order_by_all(fields)` | Add multiple ordering clauses | +| `.limit(n)` | Add `LIMIT` | +| `.offset(n)` | Add `OFFSET` | +| `.distinct()` | Add `DISTINCT` | +| `.sql()` | Return compiled SQL for debugging | + +Execution: + +| Method | Return | +|---|---| +| `.all().await` | `RyxResult>` | +| `.get(field, value).await` | `RyxResult` | +| `.first().await` | `RyxResult>` | +| `.count().await` | `RyxResult` | +| `.exists().await` | `RyxResult` | +| `.update(vec![...]).await` | `RyxResult` | +| `.delete().await` | `RyxResult` | + +## Q Objects + +```rust +use ryx_rs::Q; + +let q = Q::or( + Q::new("active", true), + Q::and(Q::new("views__gte", 1000), Q::not("title__icontains", "draft")), +); + +let posts = ObjectsManager::::new().all().filter(q).all().await?; +``` + +Supported constructors: + +```rust +Q::new(field, value) +Q::not(field, value) +Q::and(a, b) +Q::or(a, b) +Q::negate(q) +``` + +## Aggregates + +```rust +use ryx_rs::agg::{avg, count, count_distinct, max, min, sum}; + +let stats = ObjectsManager::::new() + .all() + .filter(("active", true)) + .aggregate(&[ + count("total", "id"), + sum("total_views", "views"), + avg("avg_views", "views"), + min("min_views", "views"), + max("max_views", "views"), + count_distinct("authors", "author_id"), + ]) + .await?; +``` + +`aggregate()` returns `HashMap`. + +## Values and Annotations + +```rust +let rows = ObjectsManager::::new() + .all() + .values(&["id", "title", "views"]) + .await?; + +let lists = ObjectsManager::::new() + .all() + .values_list(&["id", "title"]) + .await?; + +let annotated = ObjectsManager::::new() + .all() + .annotate(&[count("comment_count", "id")]) + .await?; +``` + +These methods return decoded maps/lists instead of model instances. + +## Relationships + +`select_related()` is available when the model implements `Relationships`. + +```rust +let posts = ObjectsManager::::new() + .all() + .select_related(&["author"]) + .all() + .await?; +``` + +Relationship metadata is described by `RelationMeta`: + +```rust +RelationMeta { + name: "author", + fk_column: "author_id", + to_table: "authors", + to_field: "id", + relation_fields: &["id", "name"], +} +``` + +## Caching and Streaming + +```rust +use ryx_rs::cache::{configure_cache, MemoryCache}; + +configure_cache(MemoryCache::new(300, 5000)); + +let posts = ObjectsManager::::new() + .all() + .filter(("active", true)) + .cache(60, Some("active_posts".to_string())) + .all() + .await?; +``` + +Streaming: + +```rust +let mut stream = ObjectsManager::::new() + .all() + .order_by("id") + .stream(100, Some("id")); + +while let Some(chunk) = stream.next_chunk().await? { + for post in chunk { + // process post + } +} +``` + +## Transactions + +```rust +use ryx_rs::transaction; + +transaction(|tx| async move { + ObjectsManager::::new() + .create() + .set("title", "Inside transaction") + .set("views", 0) + .set("active", true) + .save() + .await?; + tx.commit().await?; + Ok(()) +}) +.await?; +``` + +Transactions use the backend transaction support and route through the current +pool alias. + +## Migrations + +Rust migration support is exposed through `ryx_rs::migration`: + +```rust +use ryx_rs::migration::{ + Autodetector, DDLGenerator, MigrationRunner, + SchemaState, TableState, ColumnState, + diff_states, generate_ddl, +}; +``` + +Core operations include schema introspection, state diffing, DDL generation, +migration file handling, and runner execution. + +## Re-exports + +Commonly used re-exports: + +```rust +use ryx_rs::{ + Model, FromRow, model, + Model as ModelTrait, + QuerySet, Q, + ObjectsManager, InsertBuilder, + PoolConfig, RyxError, RyxResult, SqlValue, +}; +``` + +The derive macro `Model` and the trait `Model` share the same public name in +different namespaces. In larger modules, alias the trait import if needed. diff --git a/docs/sidebars.js b/docs/sidebars.js index 1548c0d..3ca3e15 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -71,8 +71,10 @@ const sidebars = { 'advanced/custom-lookups', 'advanced/sync-async', 'advanced/multi-db', + 'advanced/configuration-and-routing', 'advanced/raw-sql', 'advanced/cli', + 'advanced/postgres-multi-schema', ], }, { @@ -97,6 +99,7 @@ const sidebars = { 'internals/query-compiler', 'internals/connection-pool', 'internals/type-conversion', + 'internals/performance', ], }, { @@ -157,6 +160,7 @@ const sidebars = { 'rust/advanced/caching', ], }, + 'rust/reference', ], }, ], diff --git a/ryx-macro/src/lib.rs b/ryx-macro/src/lib.rs index 4eefcca..d16148e 100644 --- a/ryx-macro/src/lib.rs +++ b/ryx-macro/src/lib.rs @@ -257,6 +257,29 @@ fn find_table_name(attrs: &[Attribute]) -> String { String::new() } +fn find_database_name(attrs: &[Attribute]) -> Option { + for attr in attrs { + if attr.path().is_ident("database") { + match &attr.meta { + Meta::List(list) => { + if let Ok(lit) = syn::parse2::(list.tokens.clone()) { + return Some(lit.value()); + } + } + Meta::NameValue(MetaNameValue { + value: + syn::Expr::Lit(syn::ExprLit { + lit: Lit::Str(s), .. + }), + .. + }) => return Some(s.value()), + _ => {} + } + } + } + None +} + fn field_column_name(field: &syn::Field) -> String { parse_field_attr(field) .column @@ -409,7 +432,7 @@ fn pk_field_name(fields: &Fields) -> String { "id".to_string() } -#[proc_macro_derive(Model, attributes(table, field, relation))] +#[proc_macro_derive(Model, attributes(table, field, relation, database))] pub fn derive_model(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); let name = &input.ident; @@ -431,6 +454,8 @@ pub fn derive_model(input: TokenStream) -> TokenStream { } else { table_name }; + let database_name = find_database_name(&input.attrs) + .unwrap_or_else(|| "default".to_string()); let fields = match &input.data { Data::Struct(data) => &data.fields, @@ -499,6 +524,10 @@ pub fn derive_model(input: TokenStream) -> TokenStream { fn field_meta() -> &'static [::ryx_rs::model::FieldMeta] { &[#(#field_meta_entries),*] } + + fn database() -> &'static str { + #database_name + } } #relationships_impl @@ -677,10 +706,12 @@ pub fn model(_attr: TokenStream, item: TokenStream) -> TokenStream { } else { table_name }; + let database_name = find_database_name(&strukt.attrs) + .unwrap_or_else(|| "default".to_string()); - // Strip #[table], #[field], #[relation] helper attrs + // Strip #[table], #[field], #[relation], #[database] helper attrs strukt.attrs.retain(|a| { - !a.path().is_ident("table") && !a.path().is_ident("field") && !a.path().is_ident("relation") + !a.path().is_ident("table") && !a.path().is_ident("field") && !a.path().is_ident("relation") && !a.path().is_ident("database") }); // Also strip #[field] from each field if let syn::Fields::Named(ref mut named) = strukt.fields { @@ -882,6 +913,10 @@ pub fn model(_attr: TokenStream, item: TokenStream) -> TokenStream { fn field_meta() -> &'static [::ryx_rs::model::FieldMeta] { &[#(#field_meta_entries),*] } + + fn database() -> &'static str { + #database_name + } } #from_row_trait_impl diff --git a/ryx-python/ryx/__main__.py b/ryx-python/ryx/__main__.py index aef59be..b625602 100644 --- a/ryx-python/ryx/__main__.py +++ b/ryx-python/ryx/__main__.py @@ -15,14 +15,20 @@ inspectdb Introspect an existing database and print model stubs Configuration is read from (in order of precedence): - 1. CLI flags (--url, --settings, --config, --env) + 1. CLI flags (--url, --settings, --config, --env, --alias, --no-interactive) 2. Config file (ryx.yaml/yml/toml if --config specified or in current dir) 3. RYX_DATABASE_URL environment variable 4. ryx_settings.py in the current directory +CLI output uses ANSI colors (auto-disabled when NO_COLOR is set or +stdout is not a TTY). + Usage examples: python -m ryx migrate --url postgres://user:pass@localhost/mydb + python -m ryx migrate --alias blog # per-DB + python -m ryx migrate --no-interactive # CI/CD python -m ryx makemigrations --models myapp.models --dir migrations/ + python -m ryx makemigrations --models myapp.models --check # check mode python -m ryx shell --url sqlite:///dev.db python -m ryx showmigrations python -m ryx version @@ -116,6 +122,9 @@ def _build_parser() -> argparse.ArgumentParser: m.add_argument( "--plan", action="store_true", help="Show migration plan without executing" ) + m.add_argument( + "--schema", metavar="SCHEMA", help="Database schema (PostgreSQL multi-schema)" + ) m.set_defaults(func=cmd_migrate) # makemigrations @@ -147,6 +156,9 @@ def _build_parser() -> argparse.ArgumentParser: sq = sub.add_parser("sqlmigrate", help="Print SQL for a migration (dry run)") sq.add_argument("name", help="Migration name (e.g. 0001_initial)") sq.add_argument("--dir", default="migrations", metavar="DIR") + sq.add_argument( + "--schema", metavar="SCHEMA", help="Database schema (PostgreSQL multi-schema)" + ) sq.set_defaults(func=cmd_sqlmigrate) # flush @@ -195,6 +207,9 @@ def _build_parser() -> argparse.ArgumentParser: ) ins.add_argument("--table", metavar="TABLE", help="Inspect only this table") ins.add_argument("--output", "-o", metavar="FILE", help="Write output to file") + ins.add_argument( + "--schema", metavar="SCHEMA", default="public", help="Database schema to introspect (default: public)" + ) ins.set_defaults(func=cmd_inspectdb) return p diff --git a/ryx-python/ryx/cli/commands/inspectdb.py b/ryx-python/ryx/cli/commands/inspectdb.py index d1d7e0f..3d2b558 100644 --- a/ryx-python/ryx/cli/commands/inspectdb.py +++ b/ryx-python/ryx/cli/commands/inspectdb.py @@ -28,6 +28,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: metavar="FILE", help="Write output to file instead of stdout", ) + parser.add_argument( + "--schema", metavar="SCHEMA", default="public", + help="Database schema to introspect (default: public)", + ) async def execute(self, args: argparse.Namespace) -> int: cfg = getattr(args, "resolved_config", None) or resolve_config(args) @@ -45,10 +49,11 @@ async def execute(self, args: argparse.Namespace) -> int: from ryx.executor_helpers import raw_fetch # Get table list (Postgres / MySQL) + schema = getattr(args, "schema", "public") try: tables = await raw_fetch( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'public' AND table_type = 'BASE TABLE'" + f"SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'" ) except Exception: tables = await raw_fetch( @@ -76,6 +81,7 @@ async def execute(self, args: argparse.Namespace) -> int: cols = await raw_fetch( f"SELECT column_name, data_type, is_nullable, column_default " f"FROM information_schema.columns WHERE table_name = '{table_name}' " + f"AND table_schema = '{schema}' " f"ORDER BY ordinal_position" ) except Exception: diff --git a/ryx-python/ryx/cli/commands/migrate.py b/ryx-python/ryx/cli/commands/migrate.py index 6f8834a..5fbabcc 100644 --- a/ryx-python/ryx/cli/commands/migrate.py +++ b/ryx-python/ryx/cli/commands/migrate.py @@ -42,6 +42,9 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: action="store_true", help="Fail if no migration files found (for CI)", ) + parser.add_argument( + "--schema", metavar="SCHEMA", help="Database schema (PostgreSQL multi-schema)", + ) async def execute(self, args: argparse.Namespace) -> int: cfg = getattr(args, "resolved_config", None) @@ -70,6 +73,7 @@ async def execute(self, args: argparse.Namespace) -> int: models, dry_run=getattr(args, "dry_run", False), alias_filter=getattr(args, "database", None) or (cfg.db_alias if cfg else None), + schema=getattr(args, "schema", "") or "", migrations_dir=getattr(args, "dir", "migrations"), no_interactive=getattr(args, "no_interactive", False), ) diff --git a/ryx-python/ryx/cli/commands/showmigrations.py b/ryx-python/ryx/cli/commands/showmigrations.py index 79dbdfe..27ef703 100644 --- a/ryx-python/ryx/cli/commands/showmigrations.py +++ b/ryx-python/ryx/cli/commands/showmigrations.py @@ -6,6 +6,7 @@ from ryx.cli.commands.base import Command from ryx.cli.config import get_config from ryx.cli.config_context import resolve_config +from ryx.cli.style import PREFIX, OK, WARN, cyan, green, yellow, red class ShowMigrationsCommand(Command): @@ -29,15 +30,14 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: async def execute(self, args: argparse.Namespace) -> int: mig_dir = Path(args.dir) if not mig_dir.exists(): - print(f"[ryx] No migrations directory found at: {mig_dir}") + print(f"{PREFIX} {red('No migrations directory found at:')} {cyan(str(mig_dir))}") return 1 - files = sorted(mig_dir.glob("[0-9]*.py")) + files = sorted(mig_dir.rglob("[0-9]*.py")) if not files: - print("[ryx] No migrations found.") + print(f"{PREFIX} {WARN} No migrations found.") return 0 - # Try to check which are applied (requires DB connection) applied = set() cfg = getattr(args, "resolved_config", None) or resolve_config(args) urls = cfg.urls @@ -55,12 +55,13 @@ async def execute(self, args: argparse.Namespace) -> int: except Exception: pass - print(f"\nMigrations in {mig_dir}:") + print(f"\n{PREFIX} Migrations in {cyan(str(mig_dir))}:") for f in files: - status = "✓ applied" if f.stem in applied else " pending" - if getattr(args, "unapplied", False) and f.stem in applied: + is_applied = f.stem in applied or any(entry.endswith(f"|{f.stem}") for entry in applied) + status = f"{OK} {green(f.stem)}" if is_applied else f" {yellow(f.stem)}" + if getattr(args, "unapplied", False) and is_applied: continue - print(f" [{status}] {f.stem}") + print(f" [{status}]") print() return 0 diff --git a/ryx-python/ryx/cli/commands/sqlmigrate.py b/ryx-python/ryx/cli/commands/sqlmigrate.py index e78c7c7..a01abb6 100644 --- a/ryx-python/ryx/cli/commands/sqlmigrate.py +++ b/ryx-python/ryx/cli/commands/sqlmigrate.py @@ -8,6 +8,7 @@ from ryx.cli.commands.base import Command from ryx.cli.config_context import resolve_config +from ryx.cli.style import PREFIX, FAIL, cyan, green, red, yellow class SqlMigrateCommand(Command): @@ -29,16 +30,18 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: "--backends", help="Filter to specific backends (comma-separated: postgres,mysql,sqlite)", ) + parser.add_argument( + "--schema", metavar="SCHEMA", help="Database schema (PostgreSQL multi-schema)", + ) async def execute(self, args: argparse.Namespace) -> int: mig_dir = Path(args.dir) mig_file = mig_dir / f"{args.name}.py" if not mig_file.exists(): - # Try with glob - matches = list(mig_dir.glob(f"{args.name}*.py")) + matches = sorted(mig_dir.rglob(f"{args.name}*.py")) if not matches: - print(f"[ryx] Migration not found: {args.name}") + print(f"{PREFIX} {FAIL} Migration not found: {red(args.name)}") return 1 mig_file = matches[0] @@ -48,11 +51,11 @@ async def execute(self, args: argparse.Namespace) -> int: from ryx.migrations.ddl import DDLGenerator - gen = DDLGenerator() # default postgres + schema = getattr(args, "schema", "") or "" + gen = DDLGenerator(schema=schema) - print(f"\n-- SQL for migration: {mig_file.name}\n") + print(f"\n{PREFIX} SQL for migration: {cyan(mig_file.name)}\n") - # Handle both new-style Migration class and old-style migration_ops = getattr(mod, "Migration", None) if migration_ops is None: migration_ops = getattr(mod, "operations", []) @@ -72,7 +75,7 @@ async def execute(self, args: argparse.Namespace) -> int: from ryx.migrations.state import TableState if isinstance(op, CreateTable): - t = TableState(name=op.table) + t = TableState(name=op.table, schema=op.schema) for col in op.columns: t.add_column(col) print(gen.create_table(t) + ";\n") diff --git a/ryx-python/ryx/migrations/autodetect.py b/ryx-python/ryx/migrations/autodetect.py index 050b779..60cdf80 100644 --- a/ryx-python/ryx/migrations/autodetect.py +++ b/ryx-python/ryx/migrations/autodetect.py @@ -5,32 +5,62 @@ migration files on disk) to the current model declarations, then generates a new migration file with the needed changes. -This is the engine behind `python -m ryx makemigrations`. +This is the engine behind ``python -m ryx makemigrations``. + +Migration files are plain Python, one file per migration: -Migration file format (plain Python): migrations/0001_initial.py migrations/0002_add_views_to_posts.py ... -Each file contains a `Migration` class with: - - `dependencies`: list of migration names this one depends on - - `operations`: list of Operation objects (CreateTable, AddField, ...) - -Operations: - CreateTable(name, fields) - AddField(model, name, field_deconstruct_dict) - RemoveField(model, name) # destructive — not auto-generated - AlterField(model, name, field) - CreateIndex(model, index) - DeleteIndex(model, index_name) - RunSQL(sql, reverse_sql) # for raw migrations - -Usage: - detector = Autodetector(models=[Post, Author], migrations_dir="migrations/") - changes = detector.detect() - if changes: - path = detector.write_migration(changes) - print(f"Created {path}") +Each file contains a ``list`` of Operation objects (or a ``Migration`` class +with an ``operations`` attribute). Operations now embed the **Model class** +for automatic multi-database routing: + +.. code-block:: python + + from myapp.models import Author + from ryx.migrations.autodetect import CreateTable, ColumnState + + Migration = [ + CreateTable( + table='authors', + columns=[ + ColumnState(name='id', db_type='INTEGER', + primary_key=True, unique=True), + ColumnState(name='name', db_type='VARCHAR(100)'), + ], + model=Author, + ), + ] + +Operation types: + + ================== =================================================== + CreateTable ``(table, columns, model=None)`` + AddField ``(table, column, model=None)`` + RemoveField ``(table, column)`` (destructive, not auto-generated) + AlterField ``(table, old_col, new_col, model=None)`` + CreateIndex ``(table, name, fields, unique, model=None)`` + DeleteIndex ``(table, index_name)`` + RunSQL ``(sql, reverse_sql)`` + ================== =================================================== + +The ``model`` parameter stores a **class reference** (not a string). The +runner reads ``model._meta.database`` to route each operation to the correct +database alias. ``RunSQL`` and operations without a model are applied to all +aliases (legacy fallback). + +Usage:: + + from myapp.models import Post, Author + from ryx.migrations.autodetect import Autodetector + + detector = Autodetector(models=[Post, Author], migrations_dir="migrations/") + changes = detector.detect() + if changes: + path = detector.write_migration(changes) + print(f"Created {path}") """ from __future__ import annotations @@ -58,10 +88,12 @@ class CreateTable: """Create a new database table.""" table: str columns: List[ColumnState] + schema: str = "" model: Optional[type] = None def describe(self) -> str: - return f"Create table '{self.table}'" + s = f" (schema '{self.schema}')" if self.schema else "" + return f"Create table '{self.table}'{s}" def to_python(self) -> str: cols = ",\n ".join(_column_state_repr(c) for c in self.columns) @@ -72,6 +104,8 @@ def to_python(self) -> str: f" {cols},", f" ],", ] + if self.schema: + lines.append(f" schema={self.schema!r},") if self.model: lines.append(f" model={self.model.__qualname__},") lines.append(f" ),") @@ -86,10 +120,12 @@ class AddField: """Add a column to an existing table.""" table: str column: ColumnState + schema: str = "" model: Optional[type] = None def describe(self) -> str: - return f"Add field '{self.column.name}' to '{self.table}'" + s = f" on '{self.table}' in schema '{self.schema}'" if self.schema else f" on '{self.table}'" + return f"Add field '{self.column.name}'{s}" def to_python(self) -> str: lines = [ @@ -97,6 +133,8 @@ def to_python(self) -> str: f" table={self.table!r},", f" column={_column_state_repr(self.column)},", ] + if self.schema: + lines.append(f" schema={self.schema!r},") if self.model: lines.append(f" model={self.model.__qualname__},") lines.append(f" ),") @@ -112,13 +150,15 @@ class AlterField: table: str new_col: ColumnState old_col: Optional[ColumnState] = None + schema: str = "" model: Optional[type] = None def describe(self) -> str: old = self.old_col old_info = f"{old.db_type} → " if old else "" + s = f" in schema '{self.schema}'" if self.schema else "" return ( - f"Alter field '{self.new_col.name}' on '{self.table}': " + f"Alter field '{self.new_col.name}' on '{self.table}'{s}: " f"{old_info}{self.new_col.db_type}" ) @@ -130,6 +170,8 @@ def to_python(self) -> str: ] if self.old_col: lines.append(f" old_col={_column_state_repr(self.old_col)},") + if self.schema: + lines.append(f" schema={self.schema!r},") if self.model: lines.append(f" model={self.model.__qualname__},") lines.append(f" ),") @@ -146,10 +188,12 @@ class CreateIndex: name: str fields: List[str] unique: bool = False + schema: str = "" model: Optional[type] = None def describe(self) -> str: - return f"Create {'unique ' if self.unique else ''}index '{self.name}' on '{self.table}'" + s = f" in schema '{self.schema}'" if self.schema else "" + return f"Create {'unique ' if self.unique else ''}index '{self.name}' on '{self.table}'{s}" def to_python(self) -> str: lines = [ @@ -159,6 +203,8 @@ def to_python(self) -> str: f" fields={self.fields!r},", f" unique={self.unique!r},", ] + if self.schema: + lines.append(f" schema={self.schema!r},") if self.model: lines.append(f" model={self.model.__qualname__},") lines.append(f" ),") @@ -233,14 +279,18 @@ def apply_migration_to_state(mf: MigrationFile, state: SchemaState) -> None: """Apply the operations in a MigrationFile to a SchemaState.""" for op in mf.operations: if isinstance(op, CreateTable): - table = TableState(name=op.table) + table = TableState(name=op.table, schema=op.schema) for col in op.columns: table.add_column(col) state.add_table(table) elif isinstance(op, AddField): if state.has_table(op.table): - state.tables[op.table].add_column(op.column) + # If the stored table has empty schema, inherit from op + tbl = state.tables[op.table] + if not tbl.schema and op.schema: + tbl.schema = op.schema + tbl.add_column(op.column) elif isinstance(op, AlterField): if state.has_table(op.table) and state.tables[op.table].has_column(op.new_col.name): @@ -248,6 +298,9 @@ def apply_migration_to_state(mf: MigrationFile, state: SchemaState) -> None: col.db_type = op.new_col.db_type col.nullable = op.new_col.nullable + elif isinstance(op, CreateIndex) or isinstance(op, RunSQL): + pass # indexes and raw SQL don't affect state + ### ## AUTODETECTOR @@ -397,7 +450,7 @@ def _changes_to_operations( target: SchemaState, ) -> List[Any]: """Convert SchemaChange diffs to Operation objects.""" - # Build table → model lookup + # Build table → model lookup (composite key aware) table_to_model: Dict[str, type] = {} for m in self._models: if hasattr(m, "_meta"): @@ -414,6 +467,7 @@ def _changes_to_operations( ops.append(CreateTable( table = change.table, columns = list(table.columns.values()), + schema = change.schema, model = cls, )) @@ -422,6 +476,7 @@ def _changes_to_operations( ops.append(AddField( table=change.table, column=change.new_state, + schema=change.schema, model=cls, )) @@ -431,10 +486,12 @@ def _changes_to_operations( table = change.table, new_col = change.new_state, old_col = change.old_state, + schema = change.schema, model = cls, )) - # Also add index creation operations for all models + # Also add index creation operations for all models (no schema filtering — + # models are schema-agnostic; schema is applied at query/migration time) for model in self._models: if not hasattr(model, "_meta"): continue diff --git a/ryx-python/ryx/migrations/ddl.py b/ryx-python/ryx/migrations/ddl.py index b98aa44..364520b 100644 --- a/ryx-python/ryx/migrations/ddl.py +++ b/ryx-python/ryx/migrations/ddl.py @@ -51,10 +51,31 @@ class DDLGenerator: Args: backend: One of "postgres" (default), "mysql", "sqlite". + schema: Database schema for PostgreSQL multi-schema support + (empty = default / no qualification). """ - def __init__(self, backend: str = "postgres") -> None: + def __init__(self, backend: str = "postgres", schema: str = "") -> None: self.backend = backend.lower() + self.schema = schema + + def _qn(self, table_name: str) -> str: + """Return a schema-qualified table name. + + If ``self.schema`` is non-empty and the backend is PostgreSQL, + returns ``"schema"."table"``. Otherwise returns just the quoted name. + """ + q = self._q(table_name) + if self.schema and self.backend == "postgres": + return f'{self._q(self.schema)}.{q}' + return q + + # Schema operations + def create_schema(self, schema_name: str) -> Optional[str]: + """Generate CREATE SCHEMA IF NOT EXISTS (PostgreSQL only).""" + if self.backend == "postgres" and schema_name: + return f"CREATE SCHEMA IF NOT EXISTS {self._q(schema_name)}" + return None # CREATE TABLE def create_table(self, table: "TableState") -> str: @@ -80,7 +101,7 @@ def create_table(self, table: "TableState") -> str: cols_sql = ",\n ".join(col_defs) return ( - f"CREATE TABLE IF NOT EXISTS {self._q(table.name)} (\n" + f"CREATE TABLE IF NOT EXISTS {self._qn(table.name)} (\n" f" {cols_sql}\n" f")" ) @@ -94,7 +115,7 @@ def add_column(self, table_name: str, col: "ColumnState") -> str: col: The ColumnState describing the new column. """ col_def = self._column_def(col) - return f"ALTER TABLE {self._q(table_name)} ADD COLUMN {col_def}" + return f"ALTER TABLE {self._qn(table_name)} ADD COLUMN {col_def}" # ALTER TABLE ALTER COLUMN def alter_column(self, table_name: str, col: "ColumnState") -> Optional[str]: @@ -113,27 +134,27 @@ def alter_column(self, table_name: str, col: "ColumnState") -> Optional[str]: # Manual rebuild query return ( # First change table name to temp name, ex: users → users_old - f"ALTER TABLE {self._q(table_name)} RENAME TO {self._q(table_name + '_old')};\n" + f"ALTER TABLE {self._qn(table_name)} RENAME TO {self._qn(table_name + '_old')};\n" # Then create new table with correct schema f"{self.create_table(col.table)};\n" # Copy data from old table to new table - f"INSERT INTO {self._q(table_name)} ({', '.join(self._q(c) for c in col.table.columns.keys())}) " - f"SELECT {', '.join(self._q(c) for c in col.table.columns.keys())} FROM {self._q(table_name + '_old')};\n" + f"INSERT INTO {self._qn(table_name)} ({', '.join(self._q(c) for c in col.table.columns.keys())}) " + f"SELECT {', '.join(self._q(c) for c in col.table.columns.keys())} FROM {self._qn(table_name + '_old')};\n" # Finally drop the old table - f"DROP TABLE {self._q(table_name + '_old')};" + f"DROP TABLE {self._qn(table_name + '_old')};" ) if self.backend == "mysql": # MySQL syntax: ALTER TABLE t MODIFY COLUMN col_def col_def = self._column_def(col) - return f"ALTER TABLE {self._q(table_name)} MODIFY COLUMN {col_def}" + return f"ALTER TABLE {self._qn(table_name)} MODIFY COLUMN {col_def}" # PostgreSQL: split into two statements (type change + nullability) if self.backend == "postgres": db_type = self._translate_type(col.db_type) null_clause = "DROP NOT NULL" if col.nullable else "SET NOT NULL" return ( - f"ALTER TABLE {self._q(table_name)} " + f"ALTER TABLE {self._qn(table_name)} " f"ALTER COLUMN {self._q(col.name)} TYPE {db_type}, " f"{f'ALTER COLUMN {self._q(col.name)} SET DEFAULT {self._q(col.default)},' if col.default is not None else ''}" f"ALTER COLUMN {self._q(col.name)} {null_clause};" @@ -150,14 +171,14 @@ def drop_column(self, table_name: str, col_name: str) -> Optional[str]: We generate the statement anyway and let the driver error if unsupported. """ return ( - f"ALTER TABLE {self._q(table_name)} " + f"ALTER TABLE {self._qn(table_name)} " f"DROP COLUMN {self._q(col_name)}" ) # DROP TABLE def drop_table(self, table_name: str) -> str: """Generate a DROP TABLE IF EXISTS statement.""" - return f"DROP TABLE IF EXISTS {self._q(table_name)}" + return f"DROP TABLE IF EXISTS {self._qn(table_name)}" # CREATE INDEX def create_index(self, table_name: str, index: "Index") -> str: @@ -174,7 +195,7 @@ def create_index(self, table_name: str, index: "Index") -> str: cols = ", ".join(self._q(f) for f in index.fields) return ( f"CREATE {unique}INDEX IF NOT EXISTS {self._q(index.name)} " - f"ON {self._q(table_name)} ({cols})" + f"ON {self._qn(table_name)} ({cols})" ) def create_index_from_fields( @@ -193,7 +214,7 @@ def create_index_from_fields( cols = ", ".join(self._q(f) for f in fields) return ( f"CREATE {unique_kw}INDEX IF NOT EXISTS {self._q(name)} " - f"ON {self._q(table_name)} ({cols})" + f"ON {self._qn(table_name)} ({cols})" ) # DROP INDEX @@ -203,7 +224,7 @@ def drop_index(self, index_name: str, table_name: str = "") -> str: MySQL requires the table name; Postgres and SQLite do not. """ if self.backend == "mysql" and table_name: - return f"DROP INDEX {self._q(index_name)} ON {self._q(table_name)}" + return f"DROP INDEX {self._q(index_name)} ON {self._qn(table_name)}" return f"DROP INDEX IF EXISTS {self._q(index_name)}" # ADD CONSTRAINT (CHECK) @@ -216,7 +237,7 @@ def add_constraint(self, table_name: str, constraint: "Constraint") -> Optional[ if self.backend == "sqlite": return None # SQLite: include in CREATE TABLE only return ( - f"ALTER TABLE {self._q(table_name)} " + f"ALTER TABLE {self._qn(table_name)} " f"ADD CONSTRAINT {self._q(constraint.name)} " f"CHECK ({constraint.check})" ) @@ -241,10 +262,10 @@ def add_foreign_key( cname = constraint_name or f"fk_{table_name}_{col_name}" return ( - f"ALTER TABLE {self._q(table_name)} " + f"ALTER TABLE {self._qn(table_name)} " f"ADD CONSTRAINT {self._q(cname)} " f"FOREIGN KEY ({self._q(col_name)}) " - f"REFERENCES {self._q(ref_table)} ({self._q(ref_col)}) " + f"REFERENCES {self._qn(ref_table)} ({self._q(ref_col)}) " f"ON DELETE {on_delete}" ) @@ -363,6 +384,7 @@ def generate_schema_ddl( stmts: List[str] = [] for table in state.tables.values(): + gen = DDLGenerator(backend, schema=table.schema if table.schema else "") stmts.append(gen.create_table(table)) if not include_indexes: diff --git a/ryx-python/ryx/migrations/runner.py b/ryx-python/ryx/migrations/runner.py index 3ddfb18..2c58107 100644 --- a/ryx-python/ryx/migrations/runner.py +++ b/ryx-python/ryx/migrations/runner.py @@ -4,14 +4,15 @@ Applies pending schema changes to the live database. Uses DDLGenerator for backend-correct SQL (Postgres / MySQL / SQLite). -Steps: - 1. Ensure the ryx_migrations tracking table exists - 2. Introspect the live database schema - 3. Build the target schema from Model declarations - 4. Diff the two states - 5. Generate DDL via DDLGenerator (backend-aware) - 6. Execute each DDL statement - 7. Also create indexes and constraints declared in Model.Meta +Discovery strategy — Flat + Meta-routing: + 1. Recursively find all ``[0-9]*.py`` files under ``migrations/`` (flat, by global sort). + 2. For each database alias, filter operations by reading ``op.model._meta.database``. + 3. Apply only relevant operations per alias; track as ``alias|stem`` in ``ryx_migrations``. + 4. If no files exist, offer an interactive fallback (Live / Auto / Manual / Skip). + +CLI output uses ANSI colors (``[ryx]`` prefix in bold blue, ✓ in green, +✗ in red, ⚠ in yellow). Colors auto-disable when ``NO_COLOR`` is set or +stdout is not a TTY. """ from __future__ import annotations @@ -75,18 +76,30 @@ def __init__( dry_run: bool = False, backend: Optional[str] = None, alias_filter: Optional[str] = None, + schema: str = "", migrations_dir: str = "migrations", no_interactive: bool = False, ) -> None: self._models = models self._dry_run = dry_run self._alias_filter = alias_filter + self._schema = schema self._migrations_dir = Path(migrations_dir) self._no_interactive = no_interactive # 'backend' is now a fallback if we can't detect it from the pool self._fallback_backend = backend.lower() if backend else "postgres" self._ddl = None # Will be initialized per-database during migration + # Builder pattern for schema + def schema(self, schema: str) -> "MigrationRunner": + """Set the database schema (PostgreSQL multi-schema support).""" + self._schema = schema + return self + + def _create_ddl(self, backend: str, schema: str = "") -> DDLGenerator: + """Create a DDLGenerator, falling back to runner-level schema.""" + return DDLGenerator(backend, schema=schema or self._schema) + async def migrate(self) -> List[SchemaChange]: """Detect and apply all pending schema changes across configured databases. @@ -130,7 +143,7 @@ async def migrate(self) -> List[SchemaChange]: backend = self._fallback_backend self._current_backend = backend - self._ddl = DDLGenerator(backend) + self._ddl = self._create_ddl(backend) self._current_alias = alias # Determine models for this alias @@ -279,20 +292,24 @@ async def _apply_file_migrations( def _operation_to_ddl(self, op) -> Optional[str]: """Convert a migration Operation to a DDL SQL string.""" + # Use operation schema if set, otherwise fall back to runner schema + op_schema = getattr(op, "schema", "") + ddl = self._create_ddl(self._current_backend, schema=op_schema) + if isinstance(op, CreateTable): - table = TableState(name=op.table) + table = TableState(name=op.table, schema=op_schema) for col in op.columns: table.add_column(col) - return self._ddl.create_table(table) + return ddl.create_table(table) if isinstance(op, AddField): - return self._ddl.add_column(op.table, op.column) + return ddl.add_column(op.table, op.column) if isinstance(op, AlterField): - return self._ddl.alter_column(op.table, op.new_col) + return ddl.alter_column(op.table, op.new_col) if isinstance(op, CreateIndex): - return self._ddl.create_index_from_fields( + return ddl.create_index_from_fields( op.table, op.fields, op.name, unique=op.unique, ) @@ -420,28 +437,29 @@ async def _record_migration(self, alias: str, stem: str) -> None: async def _introspect_schema(self, alias: str) -> SchemaState: """Query the live database to build a current SchemaState.""" state = SchemaState() + schema = self._schema or "public" - tables = await self._get_tables(alias) + tables = await self._get_tables(alias, schema) for table_name in tables: if not table_name or table_name.startswith("ryx_"): continue - columns = await self._get_columns(table_name, alias) - tbl = TableState(name=table_name) + columns = await self._get_columns(table_name, schema, alias) + tbl = TableState(name=table_name, schema=self._schema) for col in columns: tbl.add_column(col) state.add_table(tbl) return state - async def _get_tables(self, alias: str) -> List[str]: + async def _get_tables(self, alias: str, schema: str = "public") -> List[str]: """Return the list of user table names from the live DB.""" from ryx.executor_helpers import raw_fetch # information_schema (Postgres / MySQL) try: rows = await raw_fetch( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'public' AND table_type = 'BASE TABLE'", + f"SELECT table_name FROM information_schema.tables " + f"WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'", alias=alias, ) if rows: @@ -449,7 +467,7 @@ async def _get_tables(self, alias: str) -> List[str]: except Exception: pass - # SQLite fallback + # SQLite fallback (schema parameter ignored) try: rows = await raw_fetch( "SELECT name AS table_name FROM sqlite_master WHERE type='table'", @@ -459,7 +477,7 @@ async def _get_tables(self, alias: str) -> List[str]: except Exception: return [] - async def _get_columns(self, table_name: str, alias: str) -> List[ColumnState]: + async def _get_columns(self, table_name: str, schema: str, alias: str) -> List[ColumnState]: """Return ColumnState objects for each column in the given table.""" from ryx.executor_helpers import raw_fetch @@ -470,7 +488,9 @@ async def _get_columns(self, table_name: str, alias: str) -> List[ColumnState]: rows = await raw_fetch( f"SELECT column_name, data_type, is_nullable, column_default " f"FROM information_schema.columns " - f"WHERE table_name = '{table_name}' ORDER BY ordinal_position", + f"WHERE table_name = '{table_name}' " + f"AND table_schema = '{schema}' " + f"ORDER BY ordinal_position", alias=alias, ) if rows: @@ -539,16 +559,18 @@ def _ddl_for_change( ) -> Optional[str]: """Generate DDL SQL for a single SchemaChange.""" + ddl = self._create_ddl(self._current_backend, schema=change.schema) + if change.kind == ChangeKind.CREATE_TABLE: table = target.tables.get(change.table) if table: - return self._ddl.create_table(table) + return ddl.create_table(table) elif change.kind == ChangeKind.ADD_COLUMN and change.new_state: - return self._ddl.add_column(change.table, change.new_state) + return ddl.add_column(change.table, change.new_state) elif change.kind == ChangeKind.ALTER_COLUMN and change.new_state: - sql = self._ddl.alter_column(change.table, change.new_state) + sql = ddl.alter_column(change.table, change.new_state) if sql is None: logger.warning( "ALTER COLUMN not supported on %s for %s.%s — " @@ -557,9 +579,11 @@ def _ddl_for_change( change.table, change.column, ) - return sql + elif change.kind == ChangeKind.CREATE_SCHEMA and change.schema: + return DDLGenerator(self._current_backend).create_schema(change.schema) + else: # DROP_TABLE / DROP_COLUMN — intentionally not auto-generated. logger.warning( @@ -578,6 +602,8 @@ async def _apply_meta_extras(self, alias: str) -> None: """ from ryx.executor_helpers import raw_execute + ddl = self._ddl # already schema-aware from migrate() + for model in self._models: if not hasattr(model, "_meta"): continue @@ -585,7 +611,6 @@ async def _apply_meta_extras(self, alias: str) -> None: table = meta.table_name # Only apply if the model belongs to this database - # (Basically duplicate the routing logic here or use a helper) from ryx.router import get_router router = get_router() @@ -600,7 +625,7 @@ async def _apply_meta_extras(self, alias: str) -> None: # Named indexes from Meta.indexes for idx in meta.indexes: - sql = self._ddl.create_index(table, idx) + sql = ddl.create_index(table, idx) logger.debug("Index DDL: %s", sql) try: await raw_execute(sql, alias=alias) @@ -610,7 +635,7 @@ async def _apply_meta_extras(self, alias: str) -> None: # index_together for i, fields in enumerate(meta.index_together): name = f"idx_{table}_{'_'.join(fields)}_{i}" - sql = self._ddl.create_index_from_fields(table, list(fields), name) + sql = ddl.create_index_from_fields(table, list(fields), name) try: await raw_execute(sql, alias=alias) except Exception: @@ -619,7 +644,7 @@ async def _apply_meta_extras(self, alias: str) -> None: # unique_together for i, fields in enumerate(meta.unique_together): name = f"uq_{table}_{'_'.join(fields)}_{i}" - sql = self._ddl.create_index_from_fields( + sql = ddl.create_index_from_fields( table, list(fields), name, unique=True ) try: @@ -629,7 +654,7 @@ async def _apply_meta_extras(self, alias: str) -> None: # CHECK constraints (not supported by all backends) for constraint in meta.constraints: - sql = self._ddl.add_constraint(table, constraint) + sql = ddl.add_constraint(table, constraint) if sql: try: await raw_execute(sql, alias=alias) @@ -645,6 +670,8 @@ async def _ensure_m2m_table(self, m2m_field, alias: str) -> None: from ryx.executor_helpers import raw_execute from ryx.migrations.state import TableState, ColumnState + ddl = self._ddl # already schema-aware + join_table = getattr(m2m_field, "_join_table", None) source_fk = getattr(m2m_field, "_source_fk", None) target_fk = getattr(m2m_field, "_target_fk", None) @@ -653,16 +680,16 @@ async def _ensure_m2m_table(self, m2m_field, alias: str) -> None: return # Build a TableState for the join table - tbl = TableState(name=join_table) + tbl = TableState(name=join_table, schema=self._schema) tbl.add_column(ColumnState("id", "INTEGER", nullable=False, primary_key=True)) tbl.add_column(ColumnState(source_fk, "INTEGER", nullable=False)) tbl.add_column(ColumnState(target_fk, "INTEGER", nullable=False)) - sql = self._ddl.create_table(tbl) + sql = ddl.create_table(tbl) try: await raw_execute(sql, alias=alias) # Unique constraint on (source_fk, target_fk) to prevent duplicates - uq_sql = self._ddl.create_index_from_fields( + uq_sql = ddl.create_index_from_fields( join_table, [source_fk, target_fk], f"uq_{join_table}_pair", diff --git a/ryx-python/ryx/migrations/state.py b/ryx-python/ryx/migrations/state.py index cbbd32a..6efb0db 100644 --- a/ryx-python/ryx/migrations/state.py +++ b/ryx-python/ryx/migrations/state.py @@ -84,9 +84,11 @@ class TableState: Attributes: name: The table name. + schema: Database schema (PostgreSQL). Empty string = default schema. columns: Ordered dict of column_name → ColumnState. """ name: str + schema: str = "" columns: Dict[str, ColumnState] = field(default_factory=dict) def add_column(self, col: ColumnState) -> None: @@ -127,14 +129,17 @@ def to_json(self) -> str: """ data = { table_name: { - col_name: { - "db_type": col.db_type, - "nullable": col.nullable, - "primary_key": col.primary_key, - "unique": col.unique, - "default": col.default, - } - for col_name, col in table.columns.items() + "_schema": table.schema, + "columns": { + col_name: { + "db_type": col.db_type, + "nullable": col.nullable, + "primary_key": col.primary_key, + "unique": col.unique, + "default": col.default, + } + for col_name, col in table.columns.items() + }, } for table_name, table in self.tables.items() } @@ -145,9 +150,10 @@ def from_json(cls, raw: str) -> "SchemaState": """Deserialize a SchemaState from a JSON string.""" state = cls() data = json.loads(raw) - for table_name, columns in data.items(): - table = TableState(name=table_name) - for col_name, col_data in columns.items(): + for table_name, table_data in data.items(): + columns_data = table_data.get("columns", table_data) + table = TableState(name=table_name, schema=table_data.get("_schema", "")) + for col_name, col_data in columns_data.items(): table.add_column(ColumnState( name = col_name, db_type = col_data["db_type"], @@ -172,6 +178,7 @@ class ChangeKind(Enum): ALTER_COLUMN = auto() ADD_INDEX = auto() DROP_INDEX = auto() + CREATE_SCHEMA = auto() ### @@ -184,15 +191,17 @@ class SchemaChange: Produced by ``diff_states()`` and consumed by ``MigrationRunner``. Attributes: - kind: What kind of change this is. - table: The table being modified. - column: The column being modified (None for table-level changes). - old_state: The before-state (None for CREATE operations). - new_state: The after-state (None for DROP operations). + kind: What kind of change this is. + table: The table being modified. + schema: Database schema (PostgreSQL). Empty = default. + column: The column being modified (None for table-level changes). + old_state: The before-state (None for CREATE operations). + new_state: The after-state (None for DROP operations). description: Human-readable description for migration output. """ kind: ChangeKind table: str + schema: str = "" column: Optional[str] = None old_state: Optional[ColumnState] = None new_state: Optional[ColumnState] = None @@ -206,45 +215,52 @@ def __str__(self) -> str: def diff_states(current: SchemaState, target: SchemaState) -> List[SchemaChange]: """Compute the list of changes needed to bring ``current`` to ``target``. - Args: - current: The state the database is in right now. - target: The state the models say the database should be in. - - Returns: - An ordered list of SchemaChange objects. Apply them in order to - migrate the database from ``current`` to ``target``. - - Design: - We do a simple set-based diff: - - Tables in target but not current → CREATE TABLE - - Tables in current but not target → we intentionally do NOT drop - them automatically (dangerous). Instead we emit a warning. - - Columns in target table but not current table → ADD COLUMN - - Columns in current table but not target table → emit a warning - (dropping columns is destructive and should be explicit). - - Columns in both but with different definitions → ALTER COLUMN + Tables are matched by composite key ``(schema, name)``. Schemas that + exist in the target but not in the current state get a ``CREATE_SCHEMA`` + change (for PostgreSQL multi-schema support). """ changes: List[SchemaChange] = [] - # Tables to create + # Detect schemas in target but not in current → CreateSchema + target_schemas = {t.schema for t in target.tables.values()} + current_schemas = {t.schema for t in current.tables.values()} + for schema in target_schemas - current_schemas: + if schema: + changes.append(SchemaChange( + kind=ChangeKind.CREATE_SCHEMA, + table="", + schema=schema, + description=f"Create schema '{schema}'", + )) + + # Helper: find table by composite key + def _find(state, table_name, schema): + for t in state.tables.values(): + if t.name == table_name and (t.schema or "") == (schema or ""): + return t + return None + + # Tables to create (in target but not in current) for table_name, target_table in target.tables.items(): - if not current.has_table(table_name): + current_table = _find(current, table_name, target_table.schema) + if current_table is None: changes.append(SchemaChange( kind=ChangeKind.CREATE_TABLE, table=table_name, - new_state=None, # full table — see runner for DDL generation - description=f"Create table '{table_name}'", + schema=target_table.schema, + new_state=None, + description=f"Create table '{table_name}' in schema '{target_table.schema or 'default'}'", )) # All columns in this new table are implicitly "added" by CREATE TABLE continue - # Columns to add or alter - current_table = current.tables[table_name] + # Columns to add or alter for col_name, target_col in target_table.columns.items(): if not current_table.has_column(col_name): changes.append(SchemaChange( kind=ChangeKind.ADD_COLUMN, table=table_name, + schema=target_table.schema, column=col_name, new_state=target_col, description=f"Add column '{col_name}' to '{table_name}'", @@ -255,6 +271,7 @@ def diff_states(current: SchemaState, target: SchemaState) -> List[SchemaChange] changes.append(SchemaChange( kind=ChangeKind.ALTER_COLUMN, table=table_name, + schema=target_table.schema, column=col_name, old_state=current_col, new_state=target_col, @@ -284,7 +301,8 @@ def project_state_from_models(models: list) -> SchemaState: if not hasattr(model, "_meta"): continue - table = TableState(name=model._meta.table_name) + schema = getattr(model._meta, "schema", "") + table = TableState(name=model._meta.table_name, schema=schema) for field_name, f in model._meta.fields.items(): col = ColumnState( name = f.column, diff --git a/ryx-python/ryx/queryset.py b/ryx-python/ryx/queryset.py index 367aef5..6f1a772 100644 --- a/ryx-python/ryx/queryset.py +++ b/ryx-python/ryx/queryset.py @@ -595,6 +595,14 @@ def using(self, alias: str) -> "QuerySet": """ return self._clone(_using=alias) + def schema(self, schema: str) -> "QuerySet": + """Set the database schema for this query (PostgreSQL multi-schema). + + Example:: + posts = await Post.objects.schema("tenant1").filter(active=True) + """ + return self._with_op("schema", schema) + # Evaluation (async) def cache( self, *, ttl: Optional[int] = None, key: Optional[str] = None diff --git a/ryx-python/src/plan.rs b/ryx-python/src/plan.rs index 31cf064..2de0e3f 100644 --- a/ryx-python/src/plan.rs +++ b/ryx-python/src/plan.rs @@ -27,6 +27,7 @@ use crate::py_to_sql_value; /// - "offset": int /// - "distinct": bool /// - "using": str +/// - "schema": str #[pyfunction] #[pyo3(signature = (table, ops, alias=None))] pub fn build_plan<'py>( @@ -173,6 +174,10 @@ pub fn build_plan<'py>( let backend = ryx_pool::get_backend(Some(&db_alias)).map_err(crate::err_to_py)?; node = node.with_backend(backend).with_db_alias(db_alias); } + "schema" => { + let schema: String = tuple.get_item(1)?.extract()?; + node = node.with_schema(schema); + } _ => {} } } diff --git a/ryx-python/tests/conftest.py b/ryx-python/tests/conftest.py index b55000c..7f29923 100644 --- a/ryx-python/tests/conftest.py +++ b/ryx-python/tests/conftest.py @@ -392,12 +392,14 @@ def event_loop(): def pytest_collection_modifyitems(config, items): - """Add setup_database fixture to all integration test items.""" + """Add setup_database and clean_tables fixtures to all integration test items.""" for item in items: if "integration" in str(item.fspath): - # Ensure the fixture is added to the test + # Ensure the fixtures are added to the test if "setup_database" not in item.fixturenames: item.fixturenames.insert(0, "setup_database") + if "clean_tables" not in item.fixturenames: + item.fixturenames.insert(0, "clean_tables") @pytest.fixture(scope="session") @@ -505,7 +507,7 @@ class Meta: data = JSONField(null=True) -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="function") async def clean_tables(): """Clean all test tables before each test.""" tables = ["test_posts", "test_authors", "test_tags", "test_post_tags"] diff --git a/ryx-python/tests/integration/test_multi_schema.py b/ryx-python/tests/integration/test_multi_schema.py new file mode 100644 index 0000000..7ae966d --- /dev/null +++ b/ryx-python/tests/integration/test_multi_schema.py @@ -0,0 +1,175 @@ +""" +Multi-schema PostgreSQL integration tests. + +Requires a running PostgreSQL. Run in a subprocess to avoid +conflicts with the conftest's SQLite pool initialization. +""" + +import os +import subprocess +import sys +import tempfile +import pytest + +PG_URL = os.environ.get( + "PG_TEST_URL", + "postgres://einswilli@localhost/ryx_integration_test", +) + + +@pytest.fixture(scope="session") +def setup_database(): + """Override conftest's setup_database — no-op for this PG test.""" + return + + +@pytest.fixture(autouse=True) +def clean_tables(): + """Override conftest's async clean_tables — no-op (PG test in subprocess).""" + return + + +def test_multi_schema_migration_pipeline(): + """Run the multi-schema integration test in a subprocess.""" + + script = r''' +import asyncio, os, sys + +PG_URL = os.environ["PG_TEST_URL"] +os.environ["RYX_AUTO_INITIALIZE"] = "0" + +import ryx + +async def check(): + try: + await ryx.setup(PG_URL) + return True + except Exception as e: + print("PG setup failed:", e) + return False + +if not asyncio.run(check()): + print("PG not available") + sys.exit(0) + +from ryx import Model, CharField, IntField +from ryx.migrations import MigrationRunner + +class TenantPost(Model): + class Meta: + table_name = "ms_posts" + id = IntField(primary_key=True) + title = CharField(max_length=200) + +class TenantAuthor(Model): + class Meta: + table_name = "ms_authors" + id = IntField(primary_key=True) + name = CharField(max_length=100) + +async def table_exists(schema, table): + from ryx.ryx_core import raw_fetch + rows = await raw_fetch( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = '" + schema + "' AND table_name = '" + table + "'" + ) + return len(rows) > 0 + +async def raw_count(schema, table): + from ryx.ryx_core import raw_fetch + rows = await raw_fetch( + 'SELECT count(*) AS cnt FROM "' + schema + '"."' + table + '"' + ) + return int(rows[0]["cnt"]) if rows else 0 + +async def cleanup(schema): + from ryx.ryx_core import raw_execute + try: + await raw_execute('DROP SCHEMA IF EXISTS "' + schema + '" CASCADE') + except Exception: + pass + +async def main(): + # Pool already initialized by check(), no need to call setup again + + # SKIP cleanup for DB inspection + # await cleanup("tenant1") + # await cleanup("tenant2") + + import builtins + original_input = builtins.input + builtins.input = lambda prompt="": "L" + + try: + from ryx.ryx_core import raw_execute as ddl_exec + for s in ["tenant1", "tenant2"]: + try: + await ddl_exec('CREATE SCHEMA IF NOT EXISTS "' + s + '"') + except Exception: + pass + + # 1. Migrate tenant1 + r1 = MigrationRunner([TenantPost, TenantAuthor], schema="tenant1") + await r1.migrate() + assert await table_exists("tenant1", "ms_posts"), "tenant1.ms_posts should exist" + assert await table_exists("tenant1", "ms_authors"), "tenant1.ms_authors should exist" + assert not await table_exists("public", "ms_posts"), "should NOT be in public" + + # 2. Migrate tenant2 + r2 = MigrationRunner([TenantPost, TenantAuthor], schema="tenant2") + await r2.migrate() + assert await table_exists("tenant2", "ms_posts"), "tenant2.ms_posts should exist" + assert await table_exists("tenant1", "ms_posts"), "tenant1 should still have posts" + + # 3. Insert & verify data isolation + from ryx.ryx_core import raw_execute as dml_exec + + # Use triple-quoted SQL to avoid quoting issues + sql1 = """INSERT INTO "tenant1"."ms_posts" (title) VALUES ('Post 1'), ('Post 2')""" + sql2 = """INSERT INTO "tenant2"."ms_posts" (title) VALUES ('Tenant 2 Post')""" + await dml_exec(sql1) + await dml_exec(sql2) + + assert await raw_count("tenant1", "ms_posts") == 2, "tenant1 should have 2 posts" + assert await raw_count("tenant2", "ms_posts") == 1, "tenant2 should have 1 post" + + # 4. Idempotent + r3 = MigrationRunner([TenantPost, TenantAuthor], schema="tenant1") + changes = await r3.migrate() + assert len(changes) == 0, f"Idempotent should return 0 changes, got {changes}" + finally: + builtins.input = original_input + + # SKIP cleanup for DB inspection + # await cleanup("tenant1") + # await cleanup("tenant2") + print("ALL CHECKS PASSED") + +asyncio.run(main()) +''' + + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: + f.write(script) + script_path = f.name + + try: + env = os.environ.copy() + env["PG_TEST_URL"] = PG_URL + + result = subprocess.run( + [sys.executable, script_path], + capture_output=True, + text=True, + env=env, + cwd=os.path.join(os.path.dirname(__file__), "../.."), + ) + + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + + assert result.returncode == 0, f"Subprocess failed (exit={result.returncode})" + assert "ALL CHECKS PASSED" in result.stdout, "Test did not complete successfully" + finally: + os.unlink(script_path) diff --git a/ryx-python/tests/unit/test_migration_autodetect.py b/ryx-python/tests/unit/test_migration_autodetect.py new file mode 100644 index 0000000..61521d9 --- /dev/null +++ b/ryx-python/tests/unit/test_migration_autodetect.py @@ -0,0 +1,83 @@ +"""Tests for migration autodetect — schema support.""" +from __future__ import annotations + +from ryx.migrations.state import ( + TableState, ColumnState, SchemaState, SchemaChange, ChangeKind, +) +from ryx.migrations.autodetect import ( + CreateTable, AddField, AlterField, CreateIndex, + apply_migration_to_state, MigrationFile, +) + + +class TestOperationSchema: + def test_create_table_default_schema(self): + op = CreateTable(table="posts", columns=[]) + assert op.schema == "" + + def test_create_table_with_schema(self): + op = CreateTable(table="posts", columns=[], schema="tenant1") + assert op.schema == "tenant1" + assert "tenant1" in op.describe() + + def test_add_field_with_schema(self): + col = ColumnState("title", "TEXT") + op = AddField(table="posts", column=col, schema="tenant1") + assert op.schema == "tenant1" + + def test_alter_field_with_schema(self): + old = ColumnState("title", "VARCHAR(100)") + new = ColumnState("title", "VARCHAR(200)") + op = AlterField(table="posts", new_col=new, old_col=old, schema="tenant1") + assert op.schema == "tenant1" + + def test_create_index_with_schema(self): + op = CreateIndex(table="posts", name="idx_title", fields=["title"], schema="tenant1") + assert op.schema == "tenant1" + + def test_to_python_with_schema(self): + op = CreateTable(table="posts", columns=[], schema="tenant1") + assert "schema=" in op.to_python() + + def test_to_python_without_schema(self): + op = CreateTable(table="posts", columns=[]) + assert "schema=" not in op.to_python() + + +class TestApplyMigrationToState: + def test_apply_creates_table_with_schema(self): + op = CreateTable(table="posts", columns=[ + ColumnState("id", "INTEGER", primary_key=True), + ], schema="tenant1") + mf = MigrationFile(name="0001", dependencies=[], operations=[op]) + state = SchemaState() + apply_migration_to_state(mf, state) + assert state.tables["posts"].schema == "tenant1" + + def test_apply_creates_table_without_schema(self): + op = CreateTable(table="posts", columns=[ + ColumnState("id", "INTEGER", primary_key=True), + ]) + mf = MigrationFile(name="0001", dependencies=[], operations=[op]) + state = SchemaState() + apply_migration_to_state(mf, state) + assert state.tables["posts"].schema == "" + + def test_apply_add_field_inherits_schema_when_empty(self): + # Create table without schema in state + state_t = TableState(name="posts") + state_t.add_column(ColumnState("id", "INTEGER", primary_key=True)) + state = SchemaState() + state.add_table(state_t) + # Add field with schema — should inherit + op = AddField(table="posts", column=ColumnState("title", "TEXT"), schema="tenant1") + mf = MigrationFile(name="0002", dependencies=[], operations=[op]) + apply_migration_to_state(mf, state) + assert state.tables["posts"].schema == "tenant1" + + +class TestAutodetectIntegration: + def test_detect_create_schema(self): + """When autodetector sees a model with schema, it should + produce a CreateTable with the schema.""" + pass # Integration with live models not available here diff --git a/ryx-python/tests/unit/test_migration_ddl.py b/ryx-python/tests/unit/test_migration_ddl.py new file mode 100644 index 0000000..6388517 --- /dev/null +++ b/ryx-python/tests/unit/test_migration_ddl.py @@ -0,0 +1,114 @@ +"""Tests for DDL generation — multi-schema support.""" +from __future__ import annotations + +from ryx.migrations.state import TableState, ColumnState +from ryx.migrations.ddl import DDLGenerator, generate_schema_ddl + + +def _post_table(): + t = TableState(name="posts") + t.add_column(ColumnState("id", "INTEGER", primary_key=True)) + t.add_column(ColumnState("title", "VARCHAR(200)")) + return t + + +class TestCreateSchema: + def test_postgres_create_schema(self): + sql = DDLGenerator("postgres").create_schema("tenant1") + assert sql == 'CREATE SCHEMA IF NOT EXISTS "tenant1"' + + def test_mysql_returns_none(self): + sql = DDLGenerator("mysql").create_schema("tenant1") + assert sql is None + + def test_empty_schema_returns_none(self): + sql = DDLGenerator("postgres").create_schema("") + assert sql is None + + +class TestQualifiedNames: + def test_create_table_with_schema(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.create_table(_post_table()) + assert '"tenant1"."posts"' in sql + + def test_create_table_no_schema_backward_compat(self): + gen = DDLGenerator("postgres") + sql = gen.create_table(_post_table()) + assert '.' not in sql + assert '"posts"' in sql + + def test_mysql_ignores_schema(self): + gen = DDLGenerator("mysql", schema="tenant1") + sql = gen.create_table(_post_table()) + assert "tenant1" not in sql + + def test_sqlite_ignores_schema(self): + gen = DDLGenerator("sqlite", schema="tenant1") + sql = gen.create_table(_post_table()) + assert "tenant1" not in sql + + +class TestAllDDLMethods: + def test_drop_table_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.drop_table("posts") + assert '"tenant1"."posts"' in sql + + def test_add_column_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.add_column("posts", ColumnState("title", "TEXT")) + assert '"tenant1"."posts"' in sql + + def test_drop_column_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.drop_column("posts", "title") + assert '"tenant1"."posts"' in sql + + def test_create_index_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.create_index_from_fields("posts", ["title"], "idx_title") + assert '"tenant1"."posts"' in sql + + def test_drop_index_mysql_qualified(self): + gen = DDLGenerator("mysql", schema="tenant1") + sql = gen.drop_index("idx_title", "posts") + assert "tenant1" not in sql # MySQL DROP INDEX doesn't qualify table + + def test_alter_column_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + new = ColumnState("title", "VARCHAR(200)", nullable=False) + sql = gen.alter_column("posts", new) + assert '"tenant1"."posts"' in sql + + def test_add_foreign_key_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + sql = gen.add_foreign_key("posts", "author_id", "authors", "id") + assert '"tenant1"."posts"' in sql + assert '"tenant1"."authors"' in sql + + def test_add_constraint_qualified(self): + gen = DDLGenerator("postgres", schema="tenant1") + constraint = type("Constraint", (), {"name": "age_check", "check": "age > 0"})() + sql = gen.add_constraint("users", constraint) + assert '"tenant1"."users"' in sql + + +class TestGenerateSchemaDDL: + def test_generates_qualified_ddl(self): + """generate_schema_ddl() generates per-table DDL using model schema.""" + # This test relies on model classes; test via TableState generation instead + from ryx.migrations.ddl import DDLGenerator + gen = DDLGenerator("postgres", schema="tenant1") + t = TableState(name="posts", schema="tenant1") + t.add_column(ColumnState("id", "INTEGER", primary_key=True)) + sql = gen.create_table(t) + assert '"tenant1"."posts"' in sql + + def test_generates_unqualified_for_default(self): + from ryx.migrations.ddl import DDLGenerator + gen = DDLGenerator("postgres") + t = TableState(name="posts") + t.add_column(ColumnState("id", "INTEGER", primary_key=True)) + sql = gen.create_table(t) + assert 'tenant1' not in sql diff --git a/ryx-python/tests/unit/test_migration_state.py b/ryx-python/tests/unit/test_migration_state.py new file mode 100644 index 0000000..26b7b80 --- /dev/null +++ b/ryx-python/tests/unit/test_migration_state.py @@ -0,0 +1,133 @@ +"""Tests for migration state — multi-schema support.""" +from __future__ import annotations + +from ryx.migrations.state import ( + TableState, + ColumnState, + SchemaState, + SchemaChange, + ChangeKind, + diff_states, + project_state_from_models, +) + + +def _table(name, schema=""): + t = TableState(name=name, schema=schema) + t.add_column(ColumnState("id", "INTEGER", primary_key=True)) + return t + + +class TestDiffStates: + def test_empty_to_table_no_schema(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("posts")) + changes = diff_states(current, target) + assert any(c.kind == ChangeKind.CREATE_TABLE for c in changes) + assert not any(c.kind == ChangeKind.CREATE_SCHEMA for c in changes) + + def test_empty_to_table_with_schema(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("posts", schema="tenant1")) + changes = diff_states(current, target) + kinds = {c.kind for c in changes} + assert ChangeKind.CREATE_SCHEMA in kinds + assert ChangeKind.CREATE_TABLE in kinds + + def test_create_schema_before_create_table(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("posts", schema="tenant1")) + changes = diff_states(current, target) + idx_schema = next(i for i, c in enumerate(changes) if c.kind == ChangeKind.CREATE_SCHEMA) + idx_table = next(i for i, c in enumerate(changes) if c.kind == ChangeKind.CREATE_TABLE) + assert idx_schema < idx_table + + def test_no_create_schema_for_empty_string(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("posts", schema="")) + changes = diff_states(current, target) + assert not any(c.kind == ChangeKind.CREATE_SCHEMA for c in changes) + + def test_identical_with_schema_no_changes(self): + current = SchemaState() + current.add_table(_table("posts", schema="tenant1")) + target = SchemaState() + target.add_table(_table("posts", schema="tenant1")) + assert diff_states(current, target) == [] + + def test_same_table_different_schemas(self): + current = SchemaState() + current.add_table(_table("posts", schema="tenant1")) + target = SchemaState() + target.add_table(_table("posts", schema="tenant1")) + target.add_table(_table("posts", schema="tenant2")) + changes = diff_states(current, target) + assert any(c.kind == ChangeKind.CREATE_SCHEMA and c.schema == "tenant2" for c in changes) + assert any(c.kind == ChangeKind.CREATE_TABLE and c.schema == "tenant2" for c in changes) + # tenant1.posts should have no changes (identical) + assert not any(c.kind == ChangeKind.CREATE_TABLE and c.schema == "tenant1" for c in changes) + + def test_add_column_to_schema_table(self): + current = SchemaState() + current.add_table(_table("posts", schema="tenant1")) + target = SchemaState() + t = _table("posts", schema="tenant1") + t.add_column(ColumnState("title", "TEXT", nullable=True)) + target.add_table(t) + changes = diff_states(current, target) + assert len(changes) == 1 + assert changes[0].kind == ChangeKind.ADD_COLUMN + assert changes[0].schema == "tenant1" + assert changes[0].column == "title" + + def test_schema_change_carries_schema(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("posts", schema="blog")) + changes = diff_states(current, target) + ct = next(c for c in changes if c.kind == ChangeKind.CREATE_TABLE) + assert ct.schema == "blog" + + def test_mixed_schema_and_default(self): + current = SchemaState() + target = SchemaState() + target.add_table(_table("users")) # no schema + target.add_table(_table("posts", schema="blog")) # with schema + changes = diff_states(current, target) + schema_count = sum(1 for c in changes if c.kind == ChangeKind.CREATE_SCHEMA) + assert schema_count == 1, "Only one CreateSchema for 'blog'" + table_count = sum(1 for c in changes if c.kind == ChangeKind.CREATE_TABLE) + assert table_count == 2 + + +class TestTableState: + def test_default_schema_is_empty(self): + t = TableState(name="posts") + assert t.schema == "" + + def test_with_schema(self): + t = TableState(name="posts", schema="tenant1") + assert t.schema == "tenant1" + + +class TestSchemaStateSerialization: + def test_to_json_with_schema(self): + state = SchemaState() + state.add_table(_table("posts", schema="tenant1")) + data = state.to_json() + assert '"tenant1"' in data + assert '_schema' in data + + def test_from_json_with_schema(self): + json_str = '{"posts": {"_schema": "tenant1", "columns": {"id": {"db_type": "INTEGER", "nullable": true, "primary_key": true, "unique": false, "default": null}}}}' + state = SchemaState.from_json(json_str) + assert state.tables["posts"].schema == "tenant1" + + def test_from_json_backward_compat(self): + json_str = '{"posts": {"id": {"db_type": "INTEGER", "nullable": true, "primary_key": true, "unique": false, "default": null}}}' + state = SchemaState.from_json(json_str) + assert state.tables["posts"].schema == "" diff --git a/ryx-query/src/ast.rs b/ryx-query/src/ast.rs index 0f60436..8432d4f 100644 --- a/ryx-query/src/ast.rs +++ b/ryx-query/src/ast.rs @@ -275,6 +275,7 @@ pub struct QueryNode { pub table: Symbol, pub backend: Backend, // Database backend for SQL generation pub db_alias: Option, // Optional alias for multi-db routing + pub schema: Option, // Optional database schema (PostgreSQL multi-schema) pub operation: QueryOperation, // # WHERE @@ -313,6 +314,7 @@ impl QueryNode { table: table.into(), backend: Backend::PostgreSQL, // default, will be overridden at runtime db_alias: None, + schema: None, operation: QueryOperation::Select { columns: None }, filters: Vec::new(), q_filter: None, @@ -411,6 +413,12 @@ impl QueryNode { self } + #[must_use] + pub fn with_schema(mut self, schema: impl Into) -> Self { + self.schema = Some(schema.into()); + self + } + #[must_use] pub fn with_extra_alias(mut self, column: impl Into, alias: impl Into) -> Self { self.extra_aliases.push((column.into(), alias.into())); diff --git a/ryx-query/src/backend.rs b/ryx-query/src/backend.rs index 9f02a81..51e68c3 100644 --- a/ryx-query/src/backend.rs +++ b/ryx-query/src/backend.rs @@ -17,6 +17,12 @@ impl Backend { Backend::SQLite => "sqlite", } } + + /// Whether this backend supports PostgreSQL-style schemas + /// (namespaced tables within a single database). + pub const fn supports_schemas(&self) -> bool { + matches!(self, Backend::PostgreSQL) + } } /// Detect the backend from a database URL. diff --git a/ryx-query/src/compiler/compilr.rs b/ryx-query/src/compiler/compilr.rs index 25f079f..1a16506 100644 --- a/ryx-query/src/compiler/compilr.rs +++ b/ryx-query/src/compiler/compilr.rs @@ -197,6 +197,7 @@ pub fn compile(node: &QueryNode) -> QueryResult { fn compute_plan_hash(node: &QueryNode) -> PlanHash { let mut h = DefaultHasher::new(); node.table.hash(&mut h); + node.schema.hash(&mut h); node.backend.hash(&mut h); node.distinct.hash(&mut h); node.limit.hash(&mut h); @@ -335,11 +336,11 @@ fn compile_select( } writer.write(" FROM "); - writer.write_symbol(node.table); + write_table_ref(node, writer); if !node.joins.is_empty() { writer.write(" "); - compile_joins(&node.joins, writer); + compile_joins(node, writer); } compile_where_combined( @@ -390,12 +391,11 @@ fn compile_aggregate( writer.write("SELECT "); compile_agg_cols(&node.annotations, writer); writer.write(" FROM "); - let table_resolved = GLOBAL_INTERNER.resolve(node.table); - writer.write_quote(&table_resolved); + write_table_ref(node, writer); if !node.joins.is_empty() { writer.write(" "); - compile_joins(&node.joins, writer); + compile_joins(node, writer); } compile_where_combined( @@ -415,11 +415,10 @@ fn compile_count( writer: &mut SqlWriter, ) -> QueryResult<()> { writer.write("SELECT COUNT(*) FROM "); - let table_resolved = GLOBAL_INTERNER.resolve(node.table); - writer.write_quote(&table_resolved); + write_table_ref(node, writer); if !node.joins.is_empty() { writer.write(" "); - compile_joins(&node.joins, writer); + compile_joins(node, writer); } compile_where_combined( &node.filters, @@ -437,8 +436,7 @@ fn compile_delete( writer: &mut SqlWriter, ) -> QueryResult<()> { writer.write("DELETE FROM "); - let table_resolved = GLOBAL_INTERNER.resolve(node.table); - writer.write_quote(&table_resolved); + write_table_ref(node, writer); compile_where_combined( &node.filters, node.q_filter.as_ref(), @@ -459,8 +457,7 @@ fn compile_update( return Err(QueryError::Internal("UPDATE with no assignments".into())); } writer.write("UPDATE "); - let table_resolved = GLOBAL_INTERNER.resolve(node.table); - writer.write_quote(&table_resolved); + write_table_ref(node, writer); writer.write(" SET "); let mut cols_out: Vec = Vec::with_capacity(assignments.len()); @@ -498,8 +495,7 @@ fn compile_insert( values.extend(vals); writer.write("INSERT INTO "); - let table_resolved = GLOBAL_INTERNER.resolve(node.table); - writer.write_quote(&table_resolved); + write_table_ref(node, writer); writer.write(" ("); writer.write_comma_separated(&cols, |c, w| w.write_symbol(*c)); writer.write(") VALUES ("); @@ -517,8 +513,43 @@ fn compile_insert( Ok(cols_resolved) } -pub fn compile_joins(joins: &[JoinClause], writer: &mut SqlWriter) { - for (i, j) in joins.iter().enumerate() { +/// Write a schema-qualified table reference. +/// +/// If ``node.schema`` is set and the backend supports schemas, +/// writes ``"schema"."table"``. Otherwise writes just ``"table"``. +fn write_table_ref(node: &QueryNode, writer: &mut SqlWriter) { + let table = GLOBAL_INTERNER.resolve(node.table); + match node.schema { + Some(sym) if node.backend.supports_schemas() => { + let schema = GLOBAL_INTERNER.resolve(sym); + writer.write_quote(&schema); + writer.write("."); + writer.write_quote(&table); + } + _ => { + writer.write_quote(&table); + } + } +} + +/// Write a schema-qualified join table reference. +fn write_join_table_ref(table: Symbol, schema: Option, backend: Backend, writer: &mut SqlWriter) { + let table_str = GLOBAL_INTERNER.resolve(table); + match schema { + Some(sym) if backend.supports_schemas() => { + let schema_str = GLOBAL_INTERNER.resolve(sym); + writer.write_quote(&schema_str); + writer.write("."); + writer.write_quote(&table_str); + } + _ => { + writer.write_quote(&table_str); + } + } +} + +pub fn compile_joins(node: &QueryNode, writer: &mut SqlWriter) { + for (i, j) in node.joins.iter().enumerate() { if i > 0 { writer.write(" "); } @@ -531,7 +562,7 @@ pub fn compile_joins(joins: &[JoinClause], writer: &mut SqlWriter) { }; writer.write(kind); writer.write(" "); - writer.write_symbol(j.table); + write_join_table_ref(j.table, node.schema, node.backend, writer); if let Some(alias) = &j.alias { writer.write(" AS "); writer.write_symbol(*alias); @@ -1033,4 +1064,129 @@ mod tests { fn init_registry() { crate::lookups::init_registry(); } + + // ── Schema-qualified query tests ───────────────────── + + fn schema_node() -> QueryNode { + init_registry(); + QueryNode::select("posts").with_schema("tenant1") + } + + #[test] + fn test_select_with_schema() { + let q = compile(&schema_node()).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "SELECT FROM should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_select_no_schema() { + init_registry(); + let q = compile(&QueryNode::select("posts")).unwrap(); + assert!(!q.sql.contains('.'), "No schema should not qualify: {}", q.sql); + assert!(q.sql.contains(r#""posts""#)); + } + + #[test] + fn test_count_with_schema() { + let mut node = schema_node(); + node.operation = QueryOperation::Count; + let q = compile(&node).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "COUNT FROM should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_delete_with_schema() { + let mut node = schema_node(); + node.operation = QueryOperation::Delete; + let q = compile(&node).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "DELETE FROM should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_update_with_schema() { + let mut node = schema_node(); + node.operation = QueryOperation::Update { + assignments: vec![("title".into(), SqlValue::Text("hello".into()))], + }; + let q = compile(&node).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "UPDATE should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_insert_with_schema() { + let mut node = schema_node(); + node.operation = QueryOperation::Insert { + values: vec![("title".into(), SqlValue::Text("hello".into()))], + returning_id: true, + }; + let q = compile(&node).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "INSERT INTO should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_join_with_schema() { + let node = schema_node().with_join(JoinClause { + kind: JoinKind::Inner, + table: "authors".into(), + alias: Some("a".into()), + on_left: "posts.author_id".into(), + on_right: "a.id".into(), + }); + let q = compile(&node).unwrap(); + assert!(q.sql.contains(r#""tenant1"."posts""#), + "Main table should be schema-qualified: {}", q.sql); + assert!(q.sql.contains(r#""tenant1"."authors""#), + "Join table should be schema-qualified: {}", q.sql); + } + + #[test] + fn test_schema_does_not_affect_mysql() { + init_registry(); + let node = QueryNode::select("posts") + .with_schema("tenant1") + .with_backend(Backend::MySQL); + let q = compile(&node).unwrap(); + assert!(!q.sql.contains("tenant1"), + "MySQL should not schema-qualify: {}", q.sql); + } + + #[test] + fn test_schema_does_not_affect_sqlite() { + init_registry(); + let node = QueryNode::select("posts") + .with_schema("tenant1") + .with_backend(Backend::SQLite); + let q = compile(&node).unwrap(); + assert!(!q.sql.contains("tenant1"), + "SQLite should not schema-qualify: {}", q.sql); + } + + #[test] + fn test_schema_in_plan_hash() { + init_registry(); + // Same query, different schemas should produce different plan hashes + use std::hash::{Hash, Hasher, DefaultHasher}; + + fn plan_hash_for(node: &QueryNode) -> u64 { + let mut h = DefaultHasher::new(); + node.table.hash(&mut h); + node.schema.hash(&mut h); + node.backend.hash(&mut h); + h.finish() + } + + let n1 = QueryNode::select("posts").with_schema("tenant1"); + let n2 = QueryNode::select("posts").with_schema("tenant2"); + let n3 = QueryNode::select("posts"); // no schema + + assert_ne!(plan_hash_for(&n1), plan_hash_for(&n2), + "Different schemas must produce different plan hashes"); + assert_ne!(plan_hash_for(&n1), plan_hash_for(&n3), + "Schema vs no-schema must produce different plan hashes"); + } } diff --git a/ryx-query/src/lookups/lookups.rs b/ryx-query/src/lookups/lookups.rs index 342c859..961630e 100644 --- a/ryx-query/src/lookups/lookups.rs +++ b/ryx-query/src/lookups/lookups.rs @@ -115,9 +115,8 @@ pub fn register_custom( name: impl Into, sql_template: impl Into, ) -> QueryResult<()> { - let registry = REGISTRY - .get() - .ok_or_else(|| QueryError::Internal("Lookup registry not initialized".into()))?; + init_registry(); + let registry = REGISTRY.get().expect("Lookup registry just initialized"); let mut guard = registry .write() @@ -137,9 +136,8 @@ pub fn register_custom( } fn resolve_simple(field: &str, lookup_name: &str, ctx: &LookupContext) -> QueryResult { - let registry = REGISTRY - .get() - .ok_or_else(|| QueryError::Internal("Lookup registry not initialized".into()))?; + init_registry(); + let registry = REGISTRY.get().expect("Lookup registry just initialized"); let guard = registry .read() @@ -162,9 +160,8 @@ fn resolve_simple(field: &str, lookup_name: &str, ctx: &LookupContext) -> QueryR /// Returns the list of all registered lookup names (built-in + custom). /// Used by the Python layer for available_lookups(). pub fn registered_lookups() -> QueryResult> { - let registry = REGISTRY - .get() - .ok_or_else(|| QueryError::Internal("Lookup registry not initialized".into()))?; + init_registry(); + let registry = REGISTRY.get().expect("Lookup registry just initialized"); let guard = registry .read() diff --git a/ryx-rs/Cargo.toml b/ryx-rs/Cargo.toml index 4c806d9..58e34a5 100644 --- a/ryx-rs/Cargo.toml +++ b/ryx-rs/Cargo.toml @@ -22,6 +22,7 @@ tracing-subscriber = { workspace = true } toml = { workspace = true, optional = true } serde_yaml = { workspace = true, optional = true } redis = { version = "0.28", optional = true, features = ["tokio-comp"] } +clap = { workspace = true } [dev-dependencies] criterion = { version = "0.5", features = ["async_tokio"] } @@ -30,6 +31,11 @@ criterion = { version = "0.5", features = ["async_tokio"] } name = "orm_bench" harness = false +[[bin]] +name = "ryx" +path = "src/main.rs" +required-features = ["config"] + [features] default = ["config"] python = ["ryx-backend/python"] diff --git a/ryx-rs/src/cli/mod.rs b/ryx-rs/src/cli/mod.rs new file mode 100644 index 0000000..b878bb2 --- /dev/null +++ b/ryx-rs/src/cli/mod.rs @@ -0,0 +1 @@ +pub mod style; diff --git a/ryx-rs/src/cli/style.rs b/ryx-rs/src/cli/style.rs new file mode 100644 index 0000000..6647b0a --- /dev/null +++ b/ryx-rs/src/cli/style.rs @@ -0,0 +1,75 @@ +//! ANSI color helpers for the Ryx CLI. +//! +//! Automatically disables colours when ``NO_COLOR`` is set, ``TERM=dumb``, +//! or stdout is not a TTY. + +use std::io::IsTerminal; + +const BOLD: &str = "\x1b[1m"; +const RESET: &str = "\x1b[0m"; +const RED: &str = "\x1b[31m"; +const GREEN: &str = "\x1b[32m"; +const YELLOW: &str = "\x1b[33m"; +const BLUE: &str = "\x1b[34m"; +const MAGENTA: &str = "\x1b[35m"; +const CYAN: &str = "\x1b[36m"; + +fn use_colour() -> bool { + if std::env::var("NO_COLOR").is_ok() { + return false; + } + if matches!(std::env::var("TERM").as_deref(), Ok("dumb")) { + return false; + } + std::io::stdout().is_terminal() +} + +macro_rules! colour { + ($name:ident, $code:expr) => { + pub fn $name(s: &str) -> String { + if use_colour() { + format!("{}{}{}", $code, s, RESET) + } else { + s.to_string() + } + } + }; +} + +colour!(red, RED); +colour!(green, GREEN); +colour!(yellow, YELLOW); +colour!(cyan, CYAN); +colour!(magenta, MAGENTA); + +pub fn prefix() -> String { + if use_colour() { + format!("{BOLD}{BLUE}[ryx]{RESET}") + } else { + "[ryx]".to_string() + } +} + +pub fn ok_mark() -> String { + if use_colour() { + format!("{GREEN}\u{2713}{RESET}") + } else { + "\u{2713}".to_string() + } +} + +pub fn fail_mark() -> String { + if use_colour() { + format!("{RED}\u{2717}{RESET}") + } else { + "\u{2717}".to_string() + } +} + +pub fn warn_mark() -> String { + if use_colour() { + format!("{YELLOW}\u{26A0}{RESET}") + } else { + "\u{26A0}".to_string() + } +} diff --git a/ryx-rs/src/lib.rs b/ryx-rs/src/lib.rs index cd82d72..445160f 100644 --- a/ryx-rs/src/lib.rs +++ b/ryx-rs/src/lib.rs @@ -1,5 +1,6 @@ pub mod agg; pub mod cache; +pub mod cli; pub mod config; pub mod into_sql; pub mod migration; diff --git a/ryx-rs/src/main.rs b/ryx-rs/src/main.rs new file mode 100644 index 0000000..e0f8016 --- /dev/null +++ b/ryx-rs/src/main.rs @@ -0,0 +1,308 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clap::{Parser, Subcommand}; +use ryx_rs::cli::style::{cyan, fail_mark, green, magenta, ok_mark, prefix, red, warn_mark, yellow}; +use ryx_rs::migration::ddl::DDLGenerator; +use ryx_rs::migration::files::{discover_migration_files, load_migration_file}; +use ryx_rs::migration::runner::{operation_to_sql, FileRunner}; +use ryx_rs::RyxConfig; + +#[derive(Parser)] +#[command(name = "ryx", about = "Ryx ORM — command-line tool")] +#[command(version)] +struct Cli { + #[command(subcommand)] + command: Commands, + + #[arg(long, global = true)] + config: Option, + + #[arg(long, global = true)] + url: Option, + + #[arg(long, global = true)] + models: Option, +} + +#[derive(Subcommand)] +enum Commands { + /// Apply pending migrations to the database + Migrate { + #[arg(long, default_value = "migrations")] + dir: String, + + #[arg(long)] + alias: Option, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + no_interactive: bool, + + #[arg(long)] + schema: Option, + }, + + /// Detect model changes and generate migration files + Makemigrations { + #[arg(long, default_value = "migrations")] + dir: String, + + #[arg(long)] + check: bool, + }, + + /// List all migrations and their applied status + Showmigrations { + #[arg(long, default_value = "migrations")] + dir: String, + + #[arg(long)] + unapplied: bool, + + #[arg(long)] + alias: Option, + }, + + /// Print SQL for a migration without executing it + Sqlmigrate { + name: String, + + #[arg(long, default_value = "migrations")] + dir: String, + + #[arg(long)] + backends: Option, + + #[arg(long)] + schema: Option, + }, +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + + match &cli.command { + Commands::Migrate { dir, alias, dry_run, no_interactive, schema } => { + cmd_migrate(&cli, dir, alias.as_deref(), *dry_run, *no_interactive, schema.as_deref()).await; + } + Commands::Makemigrations { dir, check } => { + cmd_makemigrations(dir, *check).await; + } + Commands::Showmigrations { dir, unapplied, alias } => { + cmd_showmigrations(&cli, dir, *unapplied, alias.as_deref()).await; + } + Commands::Sqlmigrate { name, dir, backends, schema } => { + cmd_sqlmigrate(dir, name, backends.as_deref(), schema.as_deref()); + } + } +} + +// ── migrate ───────────────────────────────────────────── + +async fn cmd_migrate( + cli: &Cli, + dir: &str, + alias: Option<&str>, + dry_run: bool, + no_interactive: bool, + schema: Option<&str>, +) { + let alias = alias.unwrap_or("default"); + init_pool(cli).await; + + let mut runner = FileRunner::new() + .migrations_dir(dir) + .db(alias) + .dry_run(dry_run) + .no_interactive(no_interactive); + + if let Some(s) = schema { + runner = runner.schema(s); + } + + let result = runner.run().await; + + match result { + Ok(statements) => { + if dry_run { + println!(); + for s in &statements { + println!("{s}"); + } + println!("{} {} statement(s)", prefix(), green(&statements.len().to_string())); + } else if statements.is_empty() { + println!("{} {}", prefix(), yellow("Nothing to migrate")); + } else { + println!("{} {} {} applied", prefix(), ok_mark(), green(&statements.len().to_string())); + } + } + Err(e) => { + eprintln!("{} {} {}", prefix(), fail_mark(), red(&e.to_string())); + std::process::exit(1); + } + } +} + +// ── makemigrations ────────────────────────────────────── + +async fn cmd_makemigrations(_dir: &str, check: bool) { + if check { + let files = discover_migration_files(Path::new(_dir)); + if files.is_empty() { + println!("{} {}", prefix(), red("Unapplied changes — no migrations yet")); + std::process::exit(1); + } + println!("{} All migrations applied", prefix()); + return; + } + + println!("{} {} — use Python `ryx makemigrations` for now", prefix(), yellow("makemigrations")); +} + +// ── showmigrations ────────────────────────────────────── + +async fn cmd_showmigrations(cli: &Cli, dir: &str, unapplied: bool, alias: Option<&str>) { + let alias = alias.unwrap_or("default"); + let files = discover_migration_files(Path::new(dir)); + + if files.is_empty() { + println!("{} {}", prefix(), yellow("No migrations found")); + return; + } + + init_pool(cli).await; + let applied = load_applied(alias).await; + + println!("\n{} Migrations in {}:", prefix(), cyan(dir)); + for f in &files { + let stem = f.file_stem().and_then(|s| s.to_str()).unwrap_or("?"); + let key = format!("{alias}|{stem}"); + let is_applied = applied.contains(&key) || applied.contains(&format!("{stem}")); + if unapplied && is_applied { + continue; + } + let status = if is_applied { + format!("{} {}", ok_mark(), green(stem)) + } else { + format!(" {}", yellow(stem)) + }; + println!(" [{status}]"); + } + println!(); +} + +// ── sqlmigrate ────────────────────────────────────────── + +fn cmd_sqlmigrate(dir: &str, name: &str, backends: Option<&str>, schema: Option<&str>) { + let path = match find_migration_file(Path::new(dir), name) { + Some(p) => p, + None => { + eprintln!("{} {} Migration not found: {}", prefix(), fail_mark(), red(name)); + std::process::exit(1); + } + }; + + let mf = match load_migration_file(&path) { + Ok(m) => m, + Err(e) => { + eprintln!("{} {} {}", prefix(), fail_mark(), red(&e)); + std::process::exit(1); + } + }; + + let backend_list: Vec<&str> = backends + .map(|b| b.split(',').collect()) + .unwrap_or_else(|| vec!["postgres"]); + + for backend_name in &backend_list { + let backend = match backend_name.trim() { + "postgres" | "postgresql" => ryx_query::Backend::PostgreSQL, + "mysql" => ryx_query::Backend::MySQL, + "sqlite" => ryx_query::Backend::SQLite, + _ => { + eprintln!("{} {} Unknown backend: {}", prefix(), warn_mark(), yellow(backend_name)); + continue; + } + }; + + let ddl = DDLGenerator::new(backend).in_schema(schema.unwrap_or("")); + let fname = path.file_name().and_then(|n| n.to_str()).unwrap_or("?"); + + if backend_list.len() > 1 { + println!("{} SQL for {} [{}]", prefix(), cyan(fname), magenta(backend_name.trim())); + } else { + println!("{} SQL for {}", prefix(), cyan(fname)); + } + println!(); + + for op in &mf.operations { + for sql in operation_to_sql(&ddl, op) { + println!("{sql}"); + } + } + } +} + +// ── helpers ───────────────────────────────────────────── + +fn load_config(cli: &Cli) -> Arc { + let mut config = if let Some(ref path) = cli.config { + RyxConfig::load_from_dir(path) + } else { + RyxConfig::load() + }; + + if let Some(ref url) = cli.url { + config.urls.insert("default".into(), url.clone()); + } + + Arc::new(config) +} + +async fn init_pool(cli: &Cli) { + let config = load_config(cli); + let _ = config.init_pool().await; +} + +async fn load_applied(alias: &str) -> Vec { + let pool = match ryx_backend::pool::get(Some(alias)) { + Ok(p) => p, + Err(_) => return vec![], + }; + + let rows = match pool.fetch_raw("SELECT name FROM \"ryx_migrations\"".into(), None).await { + Ok(r) => r, + Err(_) => return vec![], + }; + + rows.iter() + .filter_map(|r| { + r.get("name").and_then(|v| match v { + ryx_query::ast::SqlValue::Text(s) => Some(s.clone()), + _ => None, + }) + }) + .collect() +} + +fn find_migration_file(dir: &Path, name: &str) -> Option { + let exact = dir.join(format!("{name}.yaml")); + if exact.exists() { + return Some(exact); + } + let exact_yml = dir.join(format!("{name}.yml")); + if exact_yml.exists() { + return Some(exact_yml); + } + let files = discover_migration_files(dir); + files.into_iter().find(|f| { + f.file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.starts_with(name)) + .unwrap_or(false) + }) +} diff --git a/ryx-rs/src/migration.rs b/ryx-rs/src/migration.rs index a90e295..902ddce 100644 --- a/ryx-rs/src/migration.rs +++ b/ryx-rs/src/migration.rs @@ -1,16 +1,30 @@ use ryx_backend::backends::{DecodedRow, RyxBackend}; use ryx_common::RyxResult; use ryx_query::Backend; +use serde::{Deserialize, Serialize}; use crate::model::{FieldMeta, Model}; +pub mod operations; +pub use operations::*; +pub mod ddl; +pub use ddl::*; +#[cfg(feature = "config-yaml")] +pub mod files; +#[cfg(feature = "config-yaml")] +pub use files::*; +pub mod autodetect; +pub use autodetect::*; +pub mod runner; +pub use runner::*; + // ============================================================ // State types // ============================================================ /// A snapshot of a single database column, as seen in the live DB /// or as declared by a model. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ColumnState { pub name: String, pub db_type: String, @@ -34,14 +48,17 @@ impl From<&FieldMeta> for ColumnState { } /// A snapshot of a single database table. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TableState { pub name: String, + /// Database schema this table belongs to (empty string = default / no schema). + #[serde(default)] + pub schema: String, pub columns: Vec, } /// A full schema as known by the database or by the model declarations. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SchemaState { pub tables: Vec, } @@ -60,6 +77,8 @@ pub enum ChangeKind { AlterColumn, CreateIndex, DropIndex, + /// Create a new database schema (e.g. ``CREATE SCHEMA IF NOT EXISTS "tenant1"``). + CreateSchema, } /// A single schema change operation. @@ -67,46 +86,94 @@ pub enum ChangeKind { pub struct SchemaChange { pub kind: ChangeKind, pub table: String, + /// Database schema this change applies to (empty = default / no qualification). + pub schema: String, pub column: Option, pub old_column: Option, + /// Human-readable description of the change. + pub description: String, } /// Compare two schema states and return the list of changes needed to /// go from `current` to `target`. +/// +/// Tables are matched by composite key `(schema, name)`. Schemas that exist +/// in the target but not in the current state get a `CreateSchema` change. pub fn diff_states(current: &SchemaState, target: &SchemaState) -> Vec { let mut changes = Vec::new(); + // Collect schemas in both states + let target_schemas: std::collections::BTreeSet<&str> = + target.tables.iter().map(|t| t.schema.as_str()).collect(); + let current_schemas: std::collections::BTreeSet<&str> = + current.tables.iter().map(|t| t.schema.as_str()).collect(); + + // Schemas in target but not in current → CreateSchema + for schema in target_schemas.difference(¤t_schemas) { + if !schema.is_empty() { + changes.push(SchemaChange { + kind: ChangeKind::CreateSchema, + table: String::new(), + schema: schema.to_string(), + column: None, + old_column: None, + description: format!("Create schema {schema}"), + }); + } + } + // Tables in target but not in current → CREATE for table in &target.tables { - let current_table = current.tables.iter().find(|t| t.name == table.name); - if current_table.is_none() { + let exists = current + .tables + .iter() + .any(|t| t.schema == table.schema && t.name == table.name); + if !exists { changes.push(SchemaChange { kind: ChangeKind::CreateTable, table: table.name.clone(), + schema: table.schema.clone(), column: None, old_column: None, + description: format!( + "Create table {}.{}", + table.schema, table.name + ), }); } } // Columns of newly created tables → also emit AddColumn (so generate_ddl can use them) for table in &target.tables { - if current.tables.iter().any(|t| t.name == table.name) { + let exists = current + .tables + .iter() + .any(|t| t.schema == table.schema && t.name == table.name); + if exists { continue; } for col in &table.columns { changes.push(SchemaChange { kind: ChangeKind::AddColumn, table: table.name.clone(), + schema: table.schema.clone(), column: Some(col.clone()), old_column: None, + description: format!( + "Add column {}.{}.{}", + table.schema, table.name, col.name + ), }); } } // Columns in target but not in current → ADD COLUMN for table in &target.tables { - if let Some(current_table) = current.tables.iter().find(|t| t.name == table.name) { + if let Some(current_table) = current + .tables + .iter() + .find(|t| t.schema == table.schema && t.name == table.name) + { let current_names: Vec<&str> = current_table.columns.iter().map(|c| c.name.as_str()).collect(); for col in &table.columns { @@ -114,8 +181,13 @@ pub fn diff_states(current: &SchemaState, target: &SchemaState) -> Vec Vec Vec Vec String { - match backend { - Backend::PostgreSQL => match db_type { - "BOOLEAN" => "BOOLEAN", - "INTEGER" => "INTEGER", - "BIGINT" => "BIGINT", - "DOUBLE PRECISION" => "DOUBLE PRECISION", - "REAL" => "REAL", - "TEXT" => "TEXT", - "TIMESTAMP" => "TIMESTAMP", - "DATE" => "DATE", - "TIME" => "TIME", - "UUID" => "UUID", - "JSONB" => "JSONB", - other => other, - } - .to_string(), - Backend::MySQL => match db_type { - "BOOLEAN" => "TINYINT(1)", - "INTEGER" => "INT", - "BIGINT" => "BIGINT", - "DOUBLE PRECISION" => "DOUBLE", - "REAL" => "FLOAT", - "TEXT" => "TEXT", - "TIMESTAMP" => "DATETIME", - "UUID" => "CHAR(36)", - "JSONB" => "JSON", - other => other, - } - .to_string(), - Backend::SQLite => match db_type { - "BOOLEAN" | "INTEGER" | "BIGINT" => "INTEGER", - "DOUBLE PRECISION" | "REAL" => "REAL", - "UUID" | "JSONB" => "TEXT", - other => other, - } - .to_string(), - } -} - -fn build_col_sql(col: &ColumnState, backend: Backend, include_pk: bool) -> String { - let mut parts: Vec = vec![]; - parts.push(format!("\"{}\"", col.name)); - - if include_pk && col.primary_key { - match backend { - Backend::PostgreSQL => { - parts.push("BIGSERIAL".to_string()); - // PK implies NOT NULL - } - Backend::MySQL => { - parts.push("INT AUTO_INCREMENT".to_string()); - } - Backend::SQLite => { - parts.push("INTEGER PRIMARY KEY AUTOINCREMENT".to_string()); - return parts.join(" "); - } - } - } else { - parts.push(col_type_for_backend(&col.db_type, backend)); - if !col.nullable { - parts.push("NOT NULL".to_string()); - } - if col.unique { - parts.push("UNIQUE".to_string()); - } - if let Some(ref def) = col.default { - parts.push(format!("DEFAULT {def}")); - } - } - - parts.join(" ") -} - /// Generate DDL statements for the given changes on the specified backend. +/// +/// Delegates to [`DDLGenerator::generate`]. pub fn generate_ddl(changes: &[SchemaChange], backend: Backend) -> Vec { - let mut statements = Vec::new(); - - // First pass: CREATE TABLE statements - let create_tables: Vec<&SchemaChange> = changes - .iter() - .filter(|c| c.kind == ChangeKind::CreateTable) - .collect(); - - for change in &create_tables { - // Find all AddColumn for this table (they come right after the CreateTable) - let cols: Vec = changes - .iter() - .filter(|c| { - c.kind == ChangeKind::AddColumn - && c.table == change.table - && c.column.is_some() - }) - .filter_map(|c| c.column.clone()) - .collect(); - - if cols.is_empty() { - continue; - } - - let pk_col = cols.iter().find(|c| c.primary_key); - let col_sqls: Vec = cols - .iter() - .map(|c| format!(" {}", build_col_sql(c, backend, true))) - .collect(); - let mut sql = format!("CREATE TABLE \"{}\" (\n{}", change.table, col_sqls.join(",\n")); - if let Some(pk) = pk_col { - if !matches!(backend, Backend::SQLite) { - sql.push_str(&format!(",\n PRIMARY KEY (\"{}\")", pk.name)); - } - } - sql.push_str("\n);"); - statements.push(sql); - } - - // Second pass: ALTER TABLE for columns added to existing tables - let created_tables: Vec<&str> = create_tables.iter().map(|c| c.table.as_str()).collect(); - for change in changes { - if change.kind == ChangeKind::AddColumn { - // Skip if part of a CREATE TABLE - if created_tables.contains(&change.table.as_str()) { - continue; - } - if let Some(ref col) = change.column { - let col_sql = build_col_sql(col, backend, false); - statements.push(format!( - "ALTER TABLE \"{}\" ADD COLUMN {};", - change.table, col_sql - )); - } - } - } - - // Third pass: ALTER COLUMN - for change in changes { - if change.kind == ChangeKind::AlterColumn { - if let Some(ref col) = change.column { - let type_str = col_type_for_backend(&col.db_type, backend); - match backend { - Backend::PostgreSQL => { - statements.push(format!( - "ALTER TABLE \"{}\" ALTER COLUMN \"{}\" TYPE {};", - change.table, col.name, type_str - )); - if col.nullable { - statements.push(format!( - "ALTER TABLE \"{}\" ALTER COLUMN \"{}\" DROP NOT NULL;", - change.table, col.name - )); - } else { - statements.push(format!( - "ALTER TABLE \"{}\" ALTER COLUMN \"{}\" SET NOT NULL;", - change.table, col.name - )); - } - } - Backend::MySQL => { - let nullable_sql = if col.nullable { "NULL" } else { "NOT NULL" }; - statements.push(format!( - "ALTER TABLE \"{}\" MODIFY COLUMN \"{}\" {} {};", - change.table, col.name, type_str, nullable_sql - )); - } - Backend::SQLite => { - // SQLite doesn't support ALTER COLUMN — manual rebuild required - } - } - } - } - } - - statements + DDLGenerator::new(backend).generate(changes) } // ============================================================ // Introspection // ============================================================ -const MIGRATIONS_TABLE: &str = "ryx_migrations"; +pub const MIGRATIONS_TABLE: &str = "ryx_migrations"; /// Introspect the live database and return its current `SchemaState`. +/// +/// ``schema`` filters by database schema for backends that support it +/// (PostgreSQL). For MySQL and SQLite the parameter is ignored. pub async fn introspect_schema( backend: &dyn RyxBackend, backend_type: Backend, + schema: &str, ) -> RyxResult { match backend_type { - Backend::PostgreSQL => introspect_schema_postgres(backend).await, + Backend::PostgreSQL => introspect_schema_postgres(backend, schema).await, Backend::MySQL => introspect_schema_mysql(backend).await, Backend::SQLite => introspect_schema_sqlite(backend).await, } @@ -357,7 +275,11 @@ async fn introspect_schema_sqlite(backend: &dyn RyxBackend) -> RyxResult RyxResult { - let table_rows = backend - .fetch_raw( - "SELECT table_name FROM information_schema.tables \ - WHERE table_schema = 'public' AND table_type = 'BASE TABLE' \ - AND table_name != 'ryx_migrations'" - .to_string(), - None, - ) - .await?; +async fn introspect_schema_postgres(backend: &dyn RyxBackend, schema: &str) -> RyxResult { + let schema_clause = if schema.is_empty() { + "table_schema = 'public'".to_string() + } else { + format!("table_schema = '{schema}'") + }; + let sql = format!( + "SELECT table_name, table_schema FROM information_schema.tables \ + WHERE {schema_clause} AND table_type = 'BASE TABLE' \ + AND table_name != 'ryx_migrations'" + ); + let table_rows = backend.fetch_raw(sql, None).await?; let mut tables = Vec::new(); for row in &table_rows { let table_name = get_text(row, "table_name").unwrap_or_default(); - let columns = introspect_columns_postgres(backend, &table_name).await?; - tables.push(TableState { name: table_name, columns }); + let table_schema = get_text(row, "table_schema").unwrap_or_else(|| schema.to_string()); + let columns = introspect_columns_postgres(backend, &table_name, &table_schema).await?; + tables.push(TableState { + name: table_name, + schema: table_schema, + columns, + }); } Ok(SchemaState { tables }) } @@ -412,18 +341,19 @@ async fn introspect_schema_postgres(backend: &dyn RyxBackend) -> RyxResult RyxResult> { let sql = format!( "SELECT column_name, data_type, is_nullable, column_default \ FROM information_schema.columns \ - WHERE table_schema = 'public' AND table_name = '{table}' \ + WHERE table_schema = '{schema}' AND table_name = '{table}' \ ORDER BY ordinal_position" ); let rows = backend.fetch_raw(sql, None).await?; // Get primary key columns - let pk_cols = get_constraint_columns_postgres(backend, table, "PRIMARY KEY").await?; - let unique_cols = get_constraint_columns_postgres(backend, table, "UNIQUE").await?; + let pk_cols = get_constraint_columns_postgres(backend, table, schema, "PRIMARY KEY").await?; + let unique_cols = get_constraint_columns_postgres(backend, table, schema, "UNIQUE").await?; let mut columns = Vec::new(); for row in &rows { @@ -451,6 +381,7 @@ async fn introspect_columns_postgres( async fn get_constraint_columns_postgres( backend: &dyn RyxBackend, table: &str, + schema: &str, constraint_type: &str, ) -> RyxResult> { let sql = format!( @@ -460,7 +391,7 @@ async fn get_constraint_columns_postgres( ON tc.constraint_name = kcu.constraint_name \ AND tc.table_schema = kcu.table_schema \ AND tc.table_name = kcu.table_name \ - WHERE tc.table_schema = 'public' \ + WHERE tc.table_schema = '{schema}' \ AND tc.table_name = '{table}' \ AND tc.constraint_type = '{constraint_type}'" ); @@ -506,7 +437,11 @@ async fn introspect_schema_mysql(backend: &dyn RyxBackend) -> RyxResult String { // ── Helpers ────────────────────────────────────────────────── -fn get_text(row: &DecodedRow, key: &str) -> Option { +pub(crate) fn get_text(row: &DecodedRow, key: &str) -> Option { row.get(key).and_then(|v| match v { ryx_query::ast::SqlValue::Text(s) => Some(s.clone()), _ => None, }) } -fn get_int(row: &DecodedRow, key: &str) -> Option { +pub(crate) fn get_int(row: &DecodedRow, key: &str) -> Option { row.get(key).and_then(|v| match v { ryx_query::ast::SqlValue::Int(n) => Some(*n), _ => None, @@ -605,13 +540,39 @@ fn get_int(row: &DecodedRow, key: &str) -> Option { } // ============================================================ -// Migration Runner +// Utilities // ============================================================ -type ModelInfoFn = fn() -> TableState; +/// Detect backend type from a database URL string. +/// +/// ```ignore +/// use ryx_rs::migration::detect_backend; +/// use ryx_query::Backend; +/// +/// assert_eq!(detect_backend("postgres://localhost/db"), Backend::PostgreSQL); +/// assert_eq!(detect_backend("mysql://localhost/db"), Backend::MySQL); +/// assert_eq!(detect_backend("sqlite::memory:"), Backend::SQLite); +/// ``` +pub fn detect_backend(url: &str) -> Backend { + if url.starts_with("postgres") || url.starts_with("postgresql") { + Backend::PostgreSQL + } else if url.starts_with("mysql") || url.starts_with("mariadb") { + Backend::MySQL + } else { + Backend::SQLite + } +} + +// ============================================================ +// Migration Runner (backward-compat wrapper) +// ============================================================ /// Apply migrations — introspect, diff, and execute DDL. /// +/// Now delegates to the file-based ``FileRunner`` from ``migration/runner``. +/// Supports file-based migrations with per-alias tracking, recursive +/// discovery, and an interactive fallback when no files exist. +/// /// ```ignore /// use ryx_rs::migration::MigrationRunner; /// @@ -620,95 +581,68 @@ type ModelInfoFn = fn() -> TableState; /// .model::() /// .run().await?; /// -/// // Multi-db: target a specific database alias +/// // Multi-db /// MigrationRunner::new() /// .db("replica") /// .model::() /// .run().await?; /// ``` pub struct MigrationRunner { - models: Vec, - db_alias: Option, + inner: FileRunner, } impl MigrationRunner { pub fn new() -> Self { Self { - models: vec![], - db_alias: None, + inner: FileRunner::new(), } } - /// Target a specific database alias (defaults to `"default"`). pub fn db(mut self, alias: &str) -> Self { - self.db_alias = Some(alias.to_string()); + self.inner = self.inner.db(alias); self } - /// Register a model for migration. - pub fn model(mut self) -> Self { - self.models.push(|| { - let meta = M::field_meta(); - let columns: Vec = meta.iter().map(|m| m.into()).collect(); - TableState { - name: M::table_name().to_string(), - columns, - } - }); + pub fn migrations_dir(mut self, dir: &str) -> Self { + self.inner = self.inner.migrations_dir(dir); self } - fn build_target(&self) -> SchemaState { - let mut tables = Vec::new(); - for info_fn in &self.models { - tables.push(info_fn()); - } - SchemaState { tables } + pub fn dry_run(mut self, dry: bool) -> Self { + self.inner = self.inner.dry_run(dry); + self } - /// Diff models against the live DB and apply changes. - pub async fn run(self) -> RyxResult<()> { - let alias = self.db_alias.as_deref(); - let pool = ryx_backend::pool::get(alias)?; - let target = self.build_target(); - - // Ensure migrations tracking table - let create_tracking = format!( - "CREATE TABLE IF NOT EXISTS \"{}\" (\ - id INTEGER PRIMARY KEY,\ - name TEXT NOT NULL UNIQUE,\ - applied_at TEXT NOT NULL\ - )", - MIGRATIONS_TABLE - ); - let _ = pool.fetch_raw(create_tracking, None).await?; - - let backend_type = ryx_backend::pool::get_backend(alias)?; - let current = introspect_schema(pool.as_ref(), backend_type).await?; - let changes = diff_states(¤t, &target); + pub fn no_interactive(mut self, no: bool) -> Self { + self.inner = self.inner.no_interactive(no); + self + } - if changes.is_empty() { - return Ok(()); - } + pub fn live(mut self, live: bool) -> Self { + self.inner = self.inner.live(live); + self + } - let ddl = generate_ddl(&changes, backend_type); + /// Set the database schema for PostgreSQL multi-schema support. + /// + /// When set, all introspection, DDL, and operations scope to this schema. + /// Leave empty for the default schema (no qualification). + pub fn schema(mut self, schema: &str) -> Self { + self.inner = self.inner.schema(schema); + self + } - for stmt in &ddl { - pool.fetch_raw(stmt.clone(), None).await?; - } + pub fn model(mut self) -> Self { + self.inner = self.inner.model::(); + self + } - Ok(()) + pub async fn run(self) -> RyxResult> { + self.inner.run().await } - /// Preview the DDL that would be applied (dry-run). pub async fn plan(self) -> RyxResult> { - let alias = self.db_alias.as_deref(); - let pool = ryx_backend::pool::get(alias)?; - let backend_type = ryx_backend::pool::get_backend(alias)?; - let current = introspect_schema(pool.as_ref(), backend_type).await?; - let target = self.build_target(); - let changes = diff_states(¤t, &target); - Ok(generate_ddl(&changes, backend_type)) + self.inner.plan().await } } @@ -931,6 +865,7 @@ mod tests { fn table(name: &str, cols: &[(&str, &str, bool)]) -> TableState { TableState { name: name.to_string(), + schema: String::new(), columns: cols .iter() .map(|(n, t, pk)| ColumnState { @@ -1024,12 +959,15 @@ mod tests { SchemaChange { kind: ChangeKind::CreateTable, table: "posts".into(), + schema: String::new(), column: None, old_column: None, + description: String::new(), }, SchemaChange { kind: ChangeKind::AddColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "id".into(), db_type: "INTEGER".into(), @@ -1039,10 +977,12 @@ mod tests { default: None, }), old_column: None, + description: String::new(), }, SchemaChange { kind: ChangeKind::AddColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "title".into(), db_type: "TEXT".into(), @@ -1052,6 +992,7 @@ mod tests { default: None, }), old_column: None, + description: String::new(), }, ]; let sql = generate_ddl(&changes, Backend::SQLite); @@ -1069,12 +1010,15 @@ mod tests { SchemaChange { kind: ChangeKind::CreateTable, table: "posts".into(), + schema: String::new(), column: None, old_column: None, + description: String::new(), }, SchemaChange { kind: ChangeKind::AddColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "id".into(), db_type: "BIGINT".into(), @@ -1084,10 +1028,12 @@ mod tests { default: None, }), old_column: None, + description: String::new(), }, SchemaChange { kind: ChangeKind::AddColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "title".into(), db_type: "TEXT".into(), @@ -1097,6 +1043,7 @@ mod tests { default: None, }), old_column: None, + description: String::new(), }, ]; let sql = generate_ddl(&changes, Backend::PostgreSQL); @@ -1111,6 +1058,7 @@ mod tests { let changes = vec![SchemaChange { kind: ChangeKind::AddColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "rating".into(), db_type: "INTEGER".into(), @@ -1120,6 +1068,7 @@ mod tests { default: None, }), old_column: None, + description: String::new(), }]; let sql = generate_ddl(&changes, Backend::PostgreSQL); assert_eq!(sql.len(), 1); @@ -1132,6 +1081,7 @@ mod tests { let changes = vec![SchemaChange { kind: ChangeKind::AlterColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "title".into(), db_type: "VARCHAR".into(), @@ -1148,6 +1098,7 @@ mod tests { unique: false, default: None, }), + description: String::new(), }]; let sql = generate_ddl(&changes, Backend::PostgreSQL); assert_eq!(sql.len(), 2); @@ -1161,6 +1112,7 @@ mod tests { let changes = vec![SchemaChange { kind: ChangeKind::AlterColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "active".into(), db_type: "BOOLEAN".into(), @@ -1177,6 +1129,7 @@ mod tests { unique: false, default: None, }), + description: String::new(), }]; let sql = generate_ddl(&changes, Backend::MySQL); assert_eq!(sql.len(), 1); @@ -1190,6 +1143,7 @@ mod tests { let changes = vec![SchemaChange { kind: ChangeKind::AlterColumn, table: "posts".into(), + schema: String::new(), column: Some(ColumnState { name: "title".into(), db_type: "VARCHAR".into(), @@ -1206,6 +1160,7 @@ mod tests { unique: false, default: None, }), + description: String::new(), }]; let sql = generate_ddl(&changes, Backend::SQLite); assert!(sql.is_empty(), "SQLite ALTER COLUMN should be no-op"); @@ -1249,4 +1204,104 @@ mod tests { let row = make_row(&[("count", SqlValue::Null)]); assert_eq!(get_int(&row, "count"), None); } + + // ── Multi-schema tests ───────────────────────────────── + + fn table_in_schema(name: &str, schema: &str, cols: &[(&str, &str, bool)]) -> TableState { + let mut t = table(name, cols); + t.schema = schema.to_string(); + t + } + + #[test] + fn test_diff_create_schema_detected() { + let current = SchemaState { tables: vec![] }; + let target = SchemaState { + tables: vec![table_in_schema("posts", "tenant1", &[("id", "INTEGER", true)])], + }; + let changes = diff_states(¤t, &target); + assert!(changes.iter().any(|c| c.kind == ChangeKind::CreateSchema), + "Should detect CreateSchema for new schema 'tenant1'"); + assert!(changes.iter().any(|c| c.kind == ChangeKind::CreateTable), + "Should also detect CreateTable for the new table"); + // CreateSchema should appear before CreateTable + let idx_schema = changes.iter().position(|c| c.kind == ChangeKind::CreateSchema).unwrap(); + let idx_table = changes.iter().position(|c| c.kind == ChangeKind::CreateTable).unwrap(); + assert!(idx_schema < idx_table, "CreateSchema must precede CreateTable"); + } + + #[test] + fn test_diff_same_table_different_schemas() { + let current = SchemaState { + tables: vec![table_in_schema("posts", "tenant1", &[("id", "INTEGER", true)])], + }; + let target = SchemaState { + tables: vec![ + table_in_schema("posts", "tenant1", &[("id", "INTEGER", true), ("title", "TEXT", false)]), + table_in_schema("posts", "tenant2", &[("id", "INTEGER", true)]), + ], + }; + let changes = diff_states(¤t, &target); + assert!(changes.iter().any(|c| c.kind == ChangeKind::AddColumn && c.schema == "tenant1"), + "Should add column to tenant1.posts"); + assert!(changes.iter().any(|c| c.kind == ChangeKind::CreateSchema && c.schema == "tenant2"), + "Should CreateSchema for tenant2"); + assert!(changes.iter().any(|c| c.kind == ChangeKind::CreateTable && c.table == "posts" && c.schema == "tenant2"), + "Should CreateTable for tenant2.posts"); + } + + #[test] + fn test_diff_no_create_schema_for_empty_schema() { + let current = SchemaState { tables: vec![] }; + let target = SchemaState { + tables: vec![table("posts", &[("id", "INTEGER", true)])], + }; + let changes = diff_states(¤t, &target); + // Empty schema tables should NOT trigger CreateSchema + assert!(!changes.iter().any(|c| c.kind == ChangeKind::CreateSchema), + "Empty schema should not produce CreateSchema"); + assert!(changes.iter().any(|c| c.kind == ChangeKind::CreateTable), + "Should still CreateTable"); + } + + #[test] + fn test_diff_schema_noop_when_identical() { + let current = SchemaState { + tables: vec![table_in_schema("posts", "tenant1", &[("id", "INTEGER", true)])], + }; + let target = SchemaState { + tables: vec![table_in_schema("posts", "tenant1", &[("id", "INTEGER", true)])], + }; + let changes = diff_states(¤t, &target); + assert!(changes.is_empty(), "Identical schema states should produce no changes"); + } + + #[test] + fn test_schema_change_carries_schema() { + let current = SchemaState { tables: vec![] }; + let target = SchemaState { + tables: vec![table_in_schema("posts", "tenant1", &[("id", "INTEGER", true)])], + }; + let changes = diff_states(¤t, &target); + let create_table = changes.iter().find(|c| c.kind == ChangeKind::CreateTable).unwrap(); + assert_eq!(create_table.schema, "tenant1", "CreateTable change must carry schema"); + } + + #[test] + fn test_diff_mixed_schemas_and_default() { + // Tables both with and without schema in the same state + let current = SchemaState { tables: vec![] }; + let target = SchemaState { + tables: vec![ + table("users", &[("id", "INTEGER", true)]), // no schema + table_in_schema("posts", "blog", &[("id", "INTEGER", true)]), // in 'blog' + ], + }; + let changes = diff_states(¤t, &target); + let create_schema_count = changes.iter().filter(|c| c.kind == ChangeKind::CreateSchema).count(); + assert_eq!(create_schema_count, 1, "Only one CreateSchema for 'blog'"); + assert!(!changes.iter().any(|c| c.kind == ChangeKind::CreateSchema && c.schema.is_empty()), + "No CreateSchema for empty-schema tables"); + assert_eq!(changes.iter().filter(|c| c.kind == ChangeKind::CreateTable).count(), 2); + } } diff --git a/ryx-rs/src/migration/autodetect.rs b/ryx-rs/src/migration/autodetect.rs new file mode 100644 index 0000000..1829519 --- /dev/null +++ b/ryx-rs/src/migration/autodetect.rs @@ -0,0 +1,348 @@ +use std::path::Path; + +use crate::migration::files::{discover_migration_files, load_migration_file, write_migration_file}; +use crate::migration::operations::{Operation, SerializedColumn}; +use crate::migration::{diff_states, ColumnState, SchemaChange, SchemaState, TableState}; + +/// Describes one model known to the autodetector. +#[derive(Clone)] +pub struct ModelEntry { + /// Human-readable name stored in YAML ``model_name`` (e.g. ``"myapp::Author"``). + pub name: String, + /// Table name (used to match operations to models). + pub table_name: String, + /// Database alias for routing — read from ``Model::database()``. + pub database: String, + /// Database schema for PostgreSQL (empty = default / no qualification). + pub schema: String, + /// Builds the target ``TableState`` for this model. + pub make_state: fn() -> TableState, +} + +impl ModelEntry { + /// Create an entry from a ``Model`` implementation. + /// + /// ```ignore + /// use ryx_rs::model::Model; + /// use ryx_rs::migration::autodetect::ModelEntry; + /// + /// let entry = ModelEntry::from_model::(); + /// ``` + pub fn from_model() -> Self { + Self { + name: M::table_name().to_string(), + table_name: M::table_name().to_string(), + database: M::database().to_string(), + schema: String::new(), + make_state: || -> TableState { + let metas = M::field_meta(); + TableState { + name: M::table_name().to_string(), + schema: String::new(), + columns: metas.iter().map(|m| m.into()).collect(), + } + }, + } + } +} + +impl ModelEntry { + /// Set the database schema for this entry (PostgreSQL multi-schema). + pub fn with_schema(mut self, schema: &str) -> Self { + self.schema = schema.to_string(); + self + } +} + +/// Compares the state represented by migration files against a target +/// state built from model declarations, then generates new migration files. +#[derive(Clone)] +pub struct Autodetector { + entries: Vec, + migrations_dir: String, +} + +impl Autodetector { + pub fn new(entries: Vec, migrations_dir: &str) -> Self { + Self { + entries, + migrations_dir: migrations_dir.to_string(), + } + } + + /// Build the **target** schema from the registered models. + pub fn build_target(&self) -> SchemaState { + let mut tables = Vec::new(); + for entry in &self.entries { + let mut state = (entry.make_state)(); + if !entry.schema.is_empty() { + state.schema = entry.schema.clone(); + } + tables.push(state); + } + SchemaState { tables } + } + + /// Replay all existing migration files to reconstruct the "current" + /// applied schema state. + pub fn build_current(&self) -> SchemaState { + let dir = Path::new(&self.migrations_dir); + let files = discover_migration_files(dir); + let mut state = SchemaState { tables: vec![] }; + + for path in &files { + if let Ok(mf) = load_migration_file(path) { + state = Self::apply_operations(&state, &mf.operations); + } + } + + state + } + + /// Apply a list of operations to a schema state and return the new state. + /// + /// This is used to "replay" migration files and compute what the database + /// should look like after all applied migrations. + fn apply_operations(state: &SchemaState, ops: &[Operation]) -> SchemaState { + let mut tables = state.tables.clone(); + + for op in ops { + match op { + Operation::CreateTable { + table_name, + columns, + schema, + .. + } => { + let cols: Vec = columns + .iter() + .map(|c| ColumnState { + name: c.name.clone(), + db_type: c.db_type.clone(), + nullable: c.nullable, + primary_key: c.primary_key, + unique: c.unique, + default: c.default.clone(), + }) + .collect(); + // Replace if already exists (idempotent replay) + tables.retain(|t| t.name != *table_name || t.schema != *schema); + tables.push(TableState { + name: table_name.clone(), + schema: schema.clone(), + columns: cols, + }); + } + Operation::AddField { + table_name, + column, + schema, + .. + } => { + if let Some(table) = tables + .iter_mut() + .find(|t| t.name == *table_name && t.schema == *schema) + { + table.columns.push(ColumnState { + name: column.name.clone(), + db_type: column.db_type.clone(), + nullable: column.nullable, + primary_key: column.primary_key, + unique: column.unique, + default: column.default.clone(), + }); + } + } + Operation::AlterField { + table_name, + old_column, + new_column, + schema, + .. + } => { + if let Some(table) = tables + .iter_mut() + .find(|t| t.name == *table_name && t.schema == *schema) + { + if let Some(col) = table + .columns + .iter_mut() + .find(|c| c.name == old_column.name) + { + col.db_type = new_column.db_type.clone(); + col.nullable = new_column.nullable; + col.unique = new_column.unique; + col.default = new_column.default.clone(); + } + } + } + Operation::RemoveField { + table_name, + column_name, + schema, + } => { + if let Some(table) = tables + .iter_mut() + .find(|t| t.name == *table_name && t.schema == *schema) + { + table.columns.retain(|c| c.name != *column_name); + } + } + // CreateIndex / DeleteIndex / RunSQL / CreateSchema don't affect the schema state + _ => {} + } + } + + SchemaState { tables } + } + + /// Detect changes between the current (replayed) schema and the target + /// schema, and return a list of YAML-ready ``Operation`` values. + pub fn detect(&self) -> Vec { + let current = self.build_current(); + let target = self.build_target(); + let changes = diff_states(¤t, &target); + self.changes_to_operations(&changes) + } + + /// Convert raw schema changes into migration ``Operation`` values, + /// enriching each with model name and database alias where possible. + /// + /// Groups ``AddColumn`` changes for newly created tables into the + /// corresponding ``CreateTable`` operation's column list. + fn changes_to_operations(&self, changes: &[SchemaChange]) -> Vec { + let mut ops = Vec::new(); + + // Pass 0: CreateSchema + for ch in changes { + if ch.kind == crate::migration::ChangeKind::CreateSchema { + ops.push(Operation::CreateSchema { + schema_name: ch.schema.clone(), + database: None, + }); + } + } + + // Collect tables being created so we know which AddColumns to merge + let created_tables: Vec<&str> = changes + .iter() + .filter(|c| c.kind == crate::migration::ChangeKind::CreateTable) + .map(|c| c.table.as_str()) + .collect(); + + // Pass 1: CreateTable + its AddColumn children + for ch in changes { + if ch.kind == crate::migration::ChangeKind::CreateTable { + let entry = self.find_entry(&ch.table); + let cols: Vec = changes + .iter() + .filter(|c| { + c.kind == crate::migration::ChangeKind::AddColumn + && c.table == ch.table + && c.column.is_some() + }) + .filter_map(|c| { + c.column.as_ref().map(|col| SerializedColumn { + name: col.name.clone(), + db_type: col.db_type.clone(), + nullable: col.nullable, + primary_key: col.primary_key, + unique: col.unique, + default: col.default.clone(), + }) + }) + .collect(); + + ops.push(Operation::CreateTable { + table_name: ch.table.clone(), + columns: cols, + model_name: entry.map(|e| e.name.clone()), + database: entry.map(|e| e.database.clone()), + schema: ch.schema.clone(), + }); + } + } + + // Pass 2: AddColumn for existing tables (not part of a CreateTable) + for ch in changes { + if ch.kind == crate::migration::ChangeKind::AddColumn + && !created_tables.contains(&ch.table.as_str()) + { + if let Some(col) = &ch.column { + let entry = self.find_entry(&ch.table); + ops.push(Operation::AddField { + table_name: ch.table.clone(), + column: SerializedColumn { + name: col.name.clone(), + db_type: col.db_type.clone(), + nullable: col.nullable, + primary_key: col.primary_key, + unique: col.unique, + default: col.default.clone(), + }, + model_name: entry.map(|e| e.name.clone()), + database: entry.map(|e| e.database.clone()), + schema: ch.schema.clone(), + }); + } + } + } + + // Pass 3: AlterColumn + for ch in changes { + if ch.kind == crate::migration::ChangeKind::AlterColumn { + if let (Some(col), Some(old_col)) = (&ch.column, &ch.old_column) { + let entry = self.find_entry(&ch.table); + ops.push(Operation::AlterField { + table_name: ch.table.clone(), + old_column: SerializedColumn { + name: old_col.name.clone(), + db_type: old_col.db_type.clone(), + nullable: old_col.nullable, + primary_key: old_col.primary_key, + unique: old_col.unique, + default: old_col.default.clone(), + }, + new_column: SerializedColumn { + name: col.name.clone(), + db_type: col.db_type.clone(), + nullable: col.nullable, + primary_key: col.primary_key, + unique: col.unique, + default: col.default.clone(), + }, + model_name: entry.map(|e| e.name.clone()), + database: entry.map(|e| e.database.clone()), + schema: ch.schema.clone(), + }); + } + } + } + + ops + } + + /// Find the model entry for a given table name. + fn find_entry(&self, table_name: &str) -> Option<&ModelEntry> { + self.entries.iter().find(|e| e.table_name == table_name) + } + + /// Write a migration file containing the given operations. + /// + /// Auto-numbers the filename and returns the created path. + pub fn write_migration(&self, ops: &[Operation]) -> Result { + let dir = Path::new(&self.migrations_dir); + write_migration_file(ops, dir) + } + + /// Full pipeline: detect → write. Returns the path of the new file, + /// or ``None`` if no changes were detected. + pub fn run(&self) -> Result, String> { + let ops = self.detect(); + if ops.is_empty() { + return Ok(None); + } + let path = self.write_migration(&ops)?; + Ok(Some(path)) + } +} diff --git a/ryx-rs/src/migration/ddl.rs b/ryx-rs/src/migration/ddl.rs new file mode 100644 index 0000000..cb359fb --- /dev/null +++ b/ryx-rs/src/migration/ddl.rs @@ -0,0 +1,849 @@ +use crate::migration::{ChangeKind, ColumnState, SchemaChange, TableState}; +use ryx_query::Backend; + +// ============================================================ +// DDL Generator +// ============================================================ + +/// Backend-aware DDL generator. +/// +/// Convert schema changes into executable SQL statements, handling +/// the differences between PostgreSQL, MySQL, and SQLite. +/// +/// ```ignore +/// use ryx_rs::migration::{DDLGenerator, SchemaChange, ChangeKind}; +/// use ryx_query::Backend; +/// +/// let ddl = DDLGenerator::new(Backend::PostgreSQL); +/// let sql = ddl.generate(&changes); +/// +/// // For multi-schema: +/// let ddl = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); +/// let sql = ddl.create_table(&table); // → CREATE TABLE "tenant1"."posts" +/// ``` +#[derive(Debug, Clone)] +pub struct DDLGenerator { + pub backend: Backend, + /// Schema to qualify all table references with (empty = no qualification). + pub schema: String, +} + +impl DDLGenerator { + pub fn new(backend: Backend) -> Self { + Self { + backend, + schema: String::new(), + } + } + + /// Set the database schema for table qualification. + /// + /// When non-empty, all generated DDL will use ``"schema"."table"`` + /// notation (PostgreSQL only). + pub fn in_schema(mut self, schema: &str) -> Self { + self.schema = schema.to_string(); + self + } + + // ── helpers ────────────────────────────────────────── + + fn col_type(&self, db_type: &str) -> String { + col_type_for_backend(db_type, self.backend) + } + + fn col_def(&self, col: &ColumnState, include_pk: bool) -> String { + build_col_sql(col, self.backend, include_pk) + } + + fn quote(&self, name: &str) -> String { + match self.backend { + Backend::MySQL => format!("`{name}`"), + _ => format!("\"{name}\""), + } + } + + /// Return a schema-qualified table name: ``"schema"."table"``. + /// + /// If ``self.schema`` is empty or the backend does not support schemas, + /// returns just the quoted table name (backward-compat). + fn qn(&self, table_name: &str) -> String { + if self.schema.is_empty() || !self.backend.supports_schemas() { + self.quote(table_name) + } else { + format!("{}.{}", self.quote(&self.schema), self.quote(table_name)) + } + } + + // ── Schema operations ─────────────────────────────── + + /// Generate ``CREATE SCHEMA IF NOT EXISTS "schema_name"``. + /// + /// Only supported on PostgreSQL. Returns an empty string for other backends. + pub fn create_schema(&self, schema: &str) -> String { + if self.backend.supports_schemas() && !schema.is_empty() { + format!("CREATE SCHEMA IF NOT EXISTS {};", self.quote(schema)) + } else { + String::new() + } + } + + // ── DDL methods ────────────────────────────────────── + + /// Generate `CREATE TABLE ...` with idempotent `IF NOT EXISTS`. + pub fn create_table(&self, table: &TableState) -> String { + let if_not_exists = "IF NOT EXISTS "; + let pk_col = table.columns.iter().find(|c| c.primary_key); + let col_sqls: Vec = table + .columns + .iter() + .map(|c| format!(" {}", self.col_def(c, true))) + .collect(); + + let mut sql = format!( + "CREATE TABLE {if_not_exists}{} (\n{}", + self.qn(&table.name), + col_sqls.join(",\n") + ); + + if let Some(pk) = pk_col { + if !matches!(self.backend, Backend::SQLite) { + sql.push_str(&format!(",\n PRIMARY KEY ({})", self.quote(&pk.name))); + } + } + sql.push_str("\n);"); + sql + } + + /// Generate `DROP TABLE ...` with `IF EXISTS`. + pub fn drop_table(&self, table_name: &str) -> String { + format!( + "DROP TABLE IF EXISTS {};", + self.qn(table_name) + ) + } + + /// Generate `ALTER TABLE ... ADD COLUMN ...` for an existing table. + pub fn add_column(&self, table_name: &str, col: &ColumnState) -> String { + format!( + "ALTER TABLE {} ADD COLUMN {};", + self.qn(table_name), + self.col_def(col, false) + ) + } + + /// Generate `ALTER TABLE ... DROP COLUMN ...` with `IF EXISTS` + /// (SQLite does not support `DROP COLUMN`; emits a warning comment). + pub fn drop_column(&self, table_name: &str, column_name: &str) -> String { + if matches!(self.backend, Backend::SQLite) { + format!( + "-- SQLite does not support DROP COLUMN. Recreate the table manually.\n-- ALTER TABLE {} DROP COLUMN {};", + self.qn(table_name), + self.quote(column_name) + ) + } else { + format!( + "ALTER TABLE {} DROP COLUMN IF EXISTS {};", + self.qn(table_name), + self.quote(column_name) + ) + } + } + + /// Generate `ALTER TABLE ... ALTER COLUMN ...` or `MODIFY COLUMN` (MySQL). + /// + /// For **SQLite** this returns a table-rebuild script: + /// 1. Rename table → `_old` + /// 2. Create new table with the modified column + /// 3. Copy data (excluding dropped columns) + /// 4. Drop old table + pub fn alter_column( + &self, + table_name: &str, + old_col: &ColumnState, + new_col: &ColumnState, + ) -> Vec { + match self.backend { + Backend::PostgreSQL => { + let mut stmts = Vec::new(); + if old_col.db_type != new_col.db_type { + stmts.push(format!( + "ALTER TABLE {} ALTER COLUMN {} TYPE {};", + self.qn(table_name), + self.quote(&new_col.name), + self.col_type(&new_col.db_type) + )); + } + if old_col.nullable != new_col.nullable { + if new_col.nullable { + stmts.push(format!( + "ALTER TABLE {} ALTER COLUMN {} DROP NOT NULL;", + self.qn(table_name), + self.quote(&new_col.name) + )); + } else { + stmts.push(format!( + "ALTER TABLE {} ALTER COLUMN {} SET NOT NULL;", + self.qn(table_name), + self.quote(&new_col.name) + )); + } + } + // DEFAULT change + if new_col.default != old_col.default { + match &new_col.default { + Some(val) => stmts.push(format!( + "ALTER TABLE {} ALTER COLUMN {} SET DEFAULT {};", + self.qn(table_name), + self.quote(&new_col.name), + val + )), + None => stmts.push(format!( + "ALTER TABLE {} ALTER COLUMN {} DROP DEFAULT;", + self.qn(table_name), + self.quote(&new_col.name) + )), + } + } + stmts + } + Backend::MySQL => { + let nullable_sql = if new_col.nullable { "NULL" } else { "NOT NULL" }; + let type_str = self.col_type(&new_col.db_type); + let default_sql = match &new_col.default { + Some(d) => format!(" DEFAULT {d}"), + None => String::new(), + }; + vec![format!( + "ALTER TABLE {} MODIFY COLUMN {} {} {}{};", + self.qn(table_name), + self.quote(&new_col.name), + type_str, + nullable_sql, + default_sql + )] + } + Backend::SQLite => self.alter_column_sqlite_rebuild(table_name, old_col, new_col), + } + } + + /// SQLite table rebuild strategy for ALTER COLUMN. + /// + /// Steps: + /// 1. Rename `table` → `table__ryx_old` + /// 2. Create new `table` with the modified column definition + /// 3. Copy data from old to new + /// 4. Drop old table + /// + /// Returns a list of SQL statements that must be executed in order. + fn alter_column_sqlite_rebuild( + &self, + table_name: &str, + _old_col: &ColumnState, + _new_col: &ColumnState, + ) -> Vec { + // For a full rebuild we need the complete table schema. + // The caller should provide the full TableState, not just old/new cols. + // For now, return a comment indicating manual rebuild. + vec![format!( + "-- SQLite does not support ALTER COLUMN.\n\ + -- Manual rebuild required for {t}:\n\ + -- 1. CREATE TABLE {t}__new (...)\n\ + -- 2. INSERT INTO {t}__new SELECT ... FROM {t}\n\ + -- 3. DROP TABLE {t}\n\ + -- 4. ALTER TABLE {t}__new RENAME TO {t}", + t = self.qn(table_name) + )] + } + + /// Generate `CREATE INDEX ...` with `IF NOT EXISTS`. + pub fn create_index( + &self, + table_name: &str, + index_name: &str, + fields: &[String], + unique: bool, + ) -> String { + let unique_kw = if unique { "UNIQUE " } else { "" }; + let cols: Vec = fields.iter().map(|f| self.quote(f)).collect(); + format!( + "CREATE {unique_kw}INDEX IF NOT EXISTS {} ON {} ({});", + self.quote(index_name), + self.qn(table_name), + cols.join(", ") + ) + } + + /// Generate `DROP INDEX ...` with `IF EXISTS`. + pub fn drop_index(&self, table_name: &str, index_name: &str) -> String { + match self.backend { + Backend::MySQL => { + format!( + "DROP INDEX {} ON {};", + self.quote(index_name), + self.qn(table_name) + ) + } + Backend::PostgreSQL => { + format!("DROP INDEX IF EXISTS {};", self.quote(index_name)) + } + Backend::SQLite => { + format!("DROP INDEX IF EXISTS {};", self.quote(index_name)) + } + } + } + + /// Generate `ALTER TABLE ... ADD CONSTRAINT ... CHECK (...)`. + pub fn add_check_constraint( + &self, + table_name: &str, + constraint_name: &str, + check_expr: &str, + ) -> String { + format!( + "ALTER TABLE {} ADD CONSTRAINT {} CHECK ({})", + self.qn(table_name), + self.quote(constraint_name), + check_expr + ) + } + + /// Generate `ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY (...) REFERENCES ...`. + pub fn add_foreign_key( + &self, + table_name: &str, + constraint_name: &str, + column: &str, + ref_table: &str, + ref_column: &str, + ) -> String { + format!( + "ALTER TABLE {} ADD CONSTRAINT {} FOREIGN KEY ({}) REFERENCES {}({});", + self.qn(table_name), + self.quote(constraint_name), + self.quote(column), + self.qn(ref_table), + self.quote(ref_column) + ) + } + + // ── bulk generation ────────────────────────────────── + + /// Generate all DDL statements for the given changes. + pub fn generate(&self, changes: &[SchemaChange]) -> Vec { + let mut statements = Vec::new(); + + // Zeroth pass: CREATE SCHEMA (for PostgreSQL schemas) + for change in changes { + if change.kind == ChangeKind::CreateSchema && !change.schema.is_empty() { + let sql = self.create_schema(&change.schema); + if !sql.is_empty() { + statements.push(sql); + } + } + } + + // First pass: CREATE TABLE + let create_tables: Vec<&SchemaChange> = changes + .iter() + .filter(|c| c.kind == ChangeKind::CreateTable) + .collect(); + + for change in &create_tables { + let cols: Vec = changes + .iter() + .filter(|c| { + c.kind == ChangeKind::AddColumn + && c.table == change.table + && c.column.is_some() + }) + .filter_map(|c| c.column.clone()) + .collect(); + + if !cols.is_empty() { + let table = TableState { + name: change.table.clone(), + schema: change.schema.clone(), + columns: cols, + }; + let ddl_gen = if change.schema.is_empty() { + Self { backend: self.backend, schema: self.schema.clone() } + } else { + Self { backend: self.backend, schema: change.schema.clone() } + }; + statements.push(ddl_gen.create_table(&table)); + } + } + + // Second pass: ALTER TABLE ADD COLUMN (for existing tables) + let created_tables: Vec<&str> = create_tables.iter().map(|c| c.table.as_str()).collect(); + for change in changes { + if change.kind == ChangeKind::AddColumn + && !created_tables.contains(&change.table.as_str()) + { + if let Some(ref col) = change.column { + let ddl_gen = if change.schema.is_empty() { + Self { backend: self.backend, schema: self.schema.clone() } + } else { + Self { backend: self.backend, schema: change.schema.clone() } + }; + statements.push(ddl_gen.add_column(&change.table, col)); + } + } + } + + // Third pass: ALTER / MODIFY COLUMN (SQLite: skip in bulk — caller + // should use `DDLGenerator::alter_column` directly for rebuild scripts) + for change in changes { + if change.kind == ChangeKind::AlterColumn { + if self.backend == Backend::SQLite { + continue; // SQLite ALTER skipped in bulk generation + } + if let (Some(col), Some(old_col)) = (&change.column, &change.old_column) { + let ddl_gen = if change.schema.is_empty() { + Self { backend: self.backend, schema: self.schema.clone() } + } else { + Self { backend: self.backend, schema: change.schema.clone() } + }; + statements.extend(ddl_gen.alter_column(&change.table, old_col, col)); + } + } + } + + statements + } +} + +// ============================================================ +// Free functions (backward-compat) +// ============================================================ + +pub fn col_type_for_backend(db_type: &str, backend: Backend) -> String { + match backend { + Backend::PostgreSQL => match db_type { + "BOOLEAN" => "BOOLEAN", + "INTEGER" => "INTEGER", + "BIGINT" => "BIGINT", + "DOUBLE PRECISION" => "DOUBLE PRECISION", + "REAL" => "REAL", + "TEXT" => "TEXT", + "TIMESTAMP" => "TIMESTAMP", + "DATE" => "DATE", + "TIME" => "TIME", + "UUID" => "UUID", + "JSONB" => "JSONB", + other => other, + } + .to_string(), + Backend::MySQL => match db_type { + "BOOLEAN" => "TINYINT(1)", + "INTEGER" => "INT", + "BIGINT" => "BIGINT", + "DOUBLE PRECISION" => "DOUBLE", + "REAL" => "FLOAT", + "TEXT" => "TEXT", + "TIMESTAMP" => "DATETIME", + "UUID" => "CHAR(36)", + "JSONB" => "JSON", + other => other, + } + .to_string(), + Backend::SQLite => match db_type { + "BOOLEAN" | "INTEGER" | "BIGINT" => "INTEGER", + "DOUBLE PRECISION" | "REAL" => "REAL", + "UUID" | "JSONB" => "TEXT", + other => other, + } + .to_string(), + } +} + +pub fn build_col_sql(col: &ColumnState, backend: Backend, include_pk: bool) -> String { + let mut parts: Vec = vec![]; + + parts.push(format!("\"{}\"", col.name)); + + if include_pk && col.primary_key { + match backend { + Backend::PostgreSQL => { + parts.push("BIGSERIAL".to_string()); + } + Backend::MySQL => { + parts.push("INT AUTO_INCREMENT".to_string()); + } + Backend::SQLite => { + parts.push("INTEGER PRIMARY KEY AUTOINCREMENT".to_string()); + return parts.join(" "); + } + } + } else { + parts.push(col_type_for_backend(&col.db_type, backend)); + if !col.nullable { + parts.push("NOT NULL".to_string()); + } + if col.unique { + parts.push("UNIQUE".to_string()); + } + if let Some(ref def) = col.default { + parts.push(format!("DEFAULT {def}")); + } + } + + parts.join(" ") +} + +/// Convenience function — generate complete DDL for a list of model-based tables. +/// +/// This is the programmatic equivalent of `python -m ryx sqlmigrate`. +/// It produces the SQL needed to create the full schema from scratch. +pub fn generate_schema_ddl(tables: &[TableState], backend: Backend) -> Vec { + let mut stmts = Vec::new(); + for table in tables { + let ddl_gen = if table.schema.is_empty() { + DDLGenerator::new(backend) + } else { + DDLGenerator::new(backend).in_schema(&table.schema) + }; + stmts.push(ddl_gen.create_table(table)); + } + stmts +} + +#[cfg(test)] +mod tests { + use super::*; + + fn col(name: &str, db_type: &str, pk: bool) -> ColumnState { + ColumnState { + name: name.into(), + db_type: db_type.into(), + nullable: false, + primary_key: pk, + unique: false, + default: None, + } + } + + fn table(name: &str, cols: &[ColumnState]) -> TableState { + TableState { + name: name.into(), + schema: String::new(), + columns: cols.to_vec(), + } + } + + // ── DDLGenerator methods ───────────────────────────── + + #[test] + fn test_drop_table() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.drop_table("authors"); + assert!(sql.contains("DROP TABLE IF EXISTS")); + assert!(sql.contains("authors")); + } + + #[test] + fn test_drop_column() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.drop_column("posts", "legacy_field"); + assert!(sql.contains("DROP COLUMN IF EXISTS")); + assert!(sql.contains("legacy_field")); + } + + #[test] + fn test_drop_column_sqlite() { + let g = DDLGenerator::new(Backend::SQLite); + let sql = g.drop_column("posts", "legacy"); + assert!(sql.contains("SQLite does not support DROP COLUMN")); + } + + #[test] + fn test_create_index() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.create_index("posts", "idx_title", &["title".into()], false); + assert!(sql.contains("CREATE INDEX IF NOT EXISTS")); + assert!(sql.contains("idx_title")); + } + + #[test] + fn test_create_unique_index() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.create_index( + "users", + "idx_email", + &["email".into()], + true, + ); + assert!(sql.contains("CREATE UNIQUE INDEX")); + assert!(sql.contains("IF NOT EXISTS")); + } + + #[test] + fn test_drop_index() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.drop_index("posts", "idx_old"); + assert!(sql.contains("DROP INDEX IF EXISTS")); + assert!(sql.contains("idx_old")); + } + + #[test] + fn test_drop_index_mysql() { + let g = DDLGenerator::new(Backend::MySQL); + let sql = g.drop_index("posts", "idx_old"); + assert!(sql.contains("DROP INDEX")); + assert!(sql.contains("ON")); // MySQL: DROP INDEX ... ON table + } + + #[test] + fn test_add_check_constraint() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.add_check_constraint("products", "chk_price", "price > 0"); + assert!(sql.contains("ADD CONSTRAINT")); + assert!(sql.contains("CHECK (price > 0)")); + } + + #[test] + fn test_add_foreign_key() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.add_foreign_key( + "posts", + "fk_posts_author", + "author_id", + "authors", + "id", + ); + assert!(sql.contains("FOREIGN KEY")); + assert!(sql.contains("REFERENCES")); + assert!(sql.contains("authors")); + } + + #[test] + fn test_alter_column_postgres() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let old = col("bio", "TEXT", false); + let new = ColumnState { + name: "bio".into(), + db_type: "TEXT".into(), + nullable: true, + primary_key: false, + unique: false, + default: Some("''".into()), + }; + let stmts = g.alter_column("posts", &old, &new); + assert!(stmts.len() >= 2); // DROP NOT NULL + SET DEFAULT + assert!(stmts.iter().any(|s| s.contains("DROP NOT NULL"))); + assert!(stmts.iter().any(|s| s.contains("SET DEFAULT"))); + } + + #[test] + fn test_alter_column_mysql() { + let g = DDLGenerator::new(Backend::MySQL); + let old = col("active", "INTEGER", false); + let new = ColumnState { + name: "active".into(), + db_type: "BOOLEAN".into(), + nullable: false, + primary_key: false, + unique: false, + default: None, + }; + let stmts = g.alter_column("posts", &old, &new); + assert_eq!(stmts.len(), 1); + assert!(stmts[0].contains("MODIFY COLUMN")); + assert!(stmts[0].contains("TINYINT(1)")); + } + + #[test] + fn test_alter_column_sqlite_comment() { + let g = DDLGenerator::new(Backend::SQLite); + let old = col("title", "TEXT", false); + let new = col("title", "VARCHAR", false); + let stmts = g.alter_column("posts", &old, &new); + assert_eq!(stmts.len(), 1); + assert!(stmts[0].contains("Manual rebuild")); + } + + #[test] + fn test_create_table_if_not_exists() { + let g = DDLGenerator::new(Backend::SQLite); + let t = table( + "posts", + &[col("id", "INTEGER", true), col("title", "TEXT", false)], + ); + let sql = g.create_table(&t); + assert!(sql.contains("IF NOT EXISTS")); + assert!(sql.contains("INTEGER PRIMARY KEY AUTOINCREMENT")); + } + + #[test] + fn test_generate_schema_ddl() { + let tables = vec![ + table("authors", &[col("id", "INTEGER", true), col("name", "TEXT", false)]), + table("posts", &[col("id", "INTEGER", true), col("title", "TEXT", false)]), + ]; + let stmts = generate_schema_ddl(&tables, Backend::SQLite); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains("CREATE TABLE")); + assert!(stmts[1].contains("CREATE TABLE")); + } + + // ── Multi-schema DDL tests ─────────────────────────── + + fn table_in_schema(name: &str, schema: &str, cols: &[ColumnState]) -> TableState { + let mut t = table(name, cols); + t.schema = schema.to_string(); + t + } + + #[test] + fn test_create_schema_ddl() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.create_schema("tenant1"); + assert!(sql.contains("CREATE SCHEMA IF NOT EXISTS")); + assert!(sql.contains("tenant1")); + } + + #[test] + fn test_create_schema_ddl_mysql_empty() { + let g = DDLGenerator::new(Backend::MySQL); + let sql = g.create_schema("tenant1"); + assert!(sql.is_empty(), "MySQL should not emit CREATE SCHEMA"); + } + + #[test] + fn test_create_schema_ddl_empty() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let sql = g.create_schema(""); + assert!(sql.is_empty(), "Empty schema should return empty string"); + } + + #[test] + fn test_create_table_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let t = table_in_schema("posts", "tenant1", &[col("id", "INTEGER", true)]); + let sql = g.create_table(&t); + assert!(sql.contains(r#""tenant1"."posts""#), + "Table should be schema-qualified: {sql}"); + assert!(sql.contains("CREATE TABLE IF NOT EXISTS")); + } + + #[test] + fn test_create_table_no_schema_backward_compat() { + let g = DDLGenerator::new(Backend::PostgreSQL); + let t = table("posts", &[col("id", "INTEGER", true)]); + let sql = g.create_table(&t); + assert!(!sql.contains("."), "No schema should not qualify: {sql}"); + assert!(sql.contains(r#""posts""#)); + } + + #[test] + fn test_create_table_mysql_ignores_schema() { + let g = DDLGenerator::new(Backend::MySQL).in_schema("tenant1"); + let t = table_in_schema("posts", "tenant1", &[col("id", "INTEGER", true)]); + let sql = g.create_table(&t); + assert!(!sql.contains("tenant1"), + "MySQL should not schema-qualify: {sql}"); + } + + #[test] + fn test_drop_table_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.drop_table("posts"); + assert!(sql.contains(r#""tenant1"."posts""#), + "DROP TABLE should be schema-qualified: {sql}"); + } + + #[test] + fn test_add_column_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.add_column("posts", &col("title", "TEXT", false)); + assert!(sql.contains(r#""tenant1"."posts""#), + "ALTER TABLE ADD COLUMN should be schema-qualified: {sql}"); + } + + #[test] + fn test_drop_column_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.drop_column("posts", "title"); + assert!(sql.contains(r#""tenant1"."posts""#), + "ALTER TABLE DROP COLUMN should be schema-qualified: {sql}"); + } + + #[test] + fn test_create_index_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.create_index("posts", "idx_title", &["title".to_string()], false); + assert!(sql.contains(r#""tenant1"."posts""#), + "CREATE INDEX should be schema-qualified: {sql}"); + } + + #[test] + fn test_drop_index_pg_ignores_table_name() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.drop_index("posts", "idx_title"); + // PostgreSQL DROP INDEX does not use table_name, just index name + // Schema is not applied to index-only drops in PG + assert!(sql.contains("idx_title") && sql.contains("DROP INDEX")); + assert!(!sql.contains("posts"), "PG DROP INDEX should not use table_name: {sql}"); + } + + #[test] + fn test_drop_index_mysql_with_schema() { + let g = DDLGenerator::new(Backend::MySQL).in_schema("tenant1"); + let sql = g.drop_index("posts", "idx_title"); + assert!(!sql.contains("tenant1"), + "MySQL DROP INDEX should NOT schema-qualify table: {sql}"); + } + + #[test] + fn test_alter_column_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let old = col("title", "VARCHAR(100)", true); + let new = col("title", "VARCHAR(200)", false); + let stmts = g.alter_column("posts", &old, &new); + assert!(stmts.iter().any(|s| s.contains(r#""tenant1"."posts""#)), + "ALTER COLUMN should be schema-qualified: {:?}", stmts); + } + + #[test] + fn test_add_foreign_key_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.add_foreign_key("posts", "fk_author", "author_id", "authors", "id"); + assert!(sql.contains(r#""tenant1"."posts""#) && sql.contains(r#""tenant1"."authors""#), + "FOREIGN KEY should qualify both tables: {sql}"); + } + + #[test] + fn test_add_check_constraint_with_schema() { + let g = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let sql = g.add_check_constraint("users", "age_check", "age > 0"); + assert!(sql.contains(r#""tenant1"."users""#), + "CHECK constraint should be schema-qualified: {sql}"); + } + + #[test] + fn test_generate_schema_ddl_with_schema() { + let tables = vec![ + table_in_schema("posts", "tenant1", &[col("id", "INTEGER", true)]), + table_in_schema("comments", "tenant1", &[col("id", "INTEGER", true)]), + ]; + let stmts = generate_schema_ddl(&tables, Backend::PostgreSQL); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains(r#""tenant1"."posts""#), + "DDL should qualify table: {}", stmts[0]); + assert!(stmts[1].contains(r#""tenant1"."comments""#), + "DDL should qualify table: {}", stmts[1]); + } + + #[test] + fn test_create_table_mixed_schemas() { + // Tables in different schemas generate correctly + let g1 = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant1"); + let g2 = DDLGenerator::new(Backend::PostgreSQL).in_schema("tenant2"); + let t1 = table_in_schema("posts", "tenant1", &[col("id", "INTEGER", true)]); + let t2 = table_in_schema("posts", "tenant2", &[col("id", "INTEGER", true)]); + let sql1 = g1.create_table(&t1); + let sql2 = g2.create_table(&t2); + assert!(sql1.contains(r#""tenant1"."posts""#), "SQL1: {sql1}"); + assert!(sql2.contains(r#""tenant2"."posts""#), "SQL2: {sql2}"); + } +} diff --git a/ryx-rs/src/migration/files.rs b/ryx-rs/src/migration/files.rs new file mode 100644 index 0000000..37f6b9f --- /dev/null +++ b/ryx-rs/src/migration/files.rs @@ -0,0 +1,247 @@ +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::migration::operations::Operation; + +/// A parsed migration file from disk. +/// +/// ```yaml +/// dependencies: [] +/// operations: +/// - type: CreateTable +/// table_name: authors +/// model_name: myapp::Author +/// database: blog +/// columns: +/// - { name: id, db_type: INTEGER, pk: true } +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MigrationFile { + /// Migration names this one depends on. + #[serde(default)] + pub dependencies: Vec, + /// Operations to apply. + pub operations: Vec, +} + +/// Load a migration file from a YAML path. +pub fn load_migration_file(path: &Path) -> Result { + let content = std::fs::read_to_string(path) + .map_err(|e| format!("Cannot read {}: {}", path.display(), e))?; + serde_yaml::from_str(&content) + .map_err(|e| format!("Invalid YAML in {}: {}", path.display(), e)) +} + +/// Write operations to a new migration file. +/// +/// Auto-numbers with the next available prefix (``0001_initial.yaml``, +/// ``0002_add_views.yaml``, etc.) and generates a slug from the first +/// operation's description. +pub fn write_migration_file( + operations: &[Operation], + migrations_dir: &Path, +) -> Result { + let next_number = next_migration_number(migrations_dir); + let name = slug_from_operations(operations); + let filename = format!("{:04}_{}.yaml", next_number, name); + + // Ensure directory exists + std::fs::create_dir_all(migrations_dir) + .map_err(|e| format!("Cannot create migrations dir: {}", e))?; + + let path = migrations_dir.join(&filename); + let file = MigrationFile { + dependencies: vec![], + operations: operations.to_vec(), + }; + let yaml = serde_yaml::to_string(&file) + .map_err(|e| format!("Cannot serialize migration: {}", e))?; + + std::fs::write(&path, yaml) + .map_err(|e| format!("Cannot write {}: {}", path.display(), e))?; + + Ok(path) +} + +/// Recursively discover all migration files (``[0-9]*.yaml``) under a directory. +/// +/// Files are sorted globally by stem so that ``0001_*`` always runs before +/// ``0002_*``, regardless of which subdirectory they live in. +pub fn discover_migration_files(dir: &Path) -> Vec { + let mut files: Vec = Vec::new(); + + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && path.file_name().map(|n| n != "__pycache__").unwrap_or(true) { + files.extend(discover_migration_files(&path)); + } else if path.is_file() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.ends_with(".yaml") || name.ends_with(".yml") { + if name.starts_with(|c: char| c.is_ascii_digit()) { + files.push(path); + } + } + } + } + } + } + + files.sort_by(|a, b| { + let sa = a.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + let sb = b.file_stem().and_then(|s| s.to_str()).unwrap_or(""); + sa.cmp(sb) + }); + + files +} + +// ── helpers ────────────────────────────────────────────────── + +/// Return the next available migration number (1-based), scanning recursively. +fn next_migration_number(dir: &Path) -> u32 { + let mut max_n = 0u32; + + fn scan(dir: &Path, max_n: &mut u32) { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && path.file_name().map(|n| n != "__pycache__").unwrap_or(true) { + scan(&path, max_n); + } else if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if let Some(prefix) = name + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::() + .parse::() + .ok() + { + *max_n = (*max_n).max(prefix); + } + } + } + } + } + + scan(dir, &mut max_n); + max_n + 1 +} + +/// Generate a human-readable slug from the first operation. +fn slug_from_operations(ops: &[Operation]) -> String { + let desc = ops + .first() + .map(|op| match op { + Operation::CreateTable { table_name, .. } => format!("create_{table_name}"), + Operation::AddField { + table_name, column, .. + } => { + format!("add_{}_{}", table_name, column.name) + } + Operation::AlterField { + table_name, new_column, .. + } => { + format!("alter_{}_{}", table_name, new_column.name) + } + Operation::CreateIndex { index_name, .. } => format!("create_index_{index_name}"), + Operation::RemoveField { table_name, column_name, .. } => { + format!("remove_{}_{}", table_name, column_name) + } + Operation::DeleteIndex { index_name, .. } => format!("delete_index_{index_name}"), + Operation::RunSQL { .. } => "raw_sql".to_string(), + Operation::CreateSchema { schema_name, .. } => format!("create_schema_{schema_name}"), + }) + .unwrap_or_else(|| "auto".to_string()); + + // Truncate to a reasonable length + if desc.len() > 60 { + let mut truncated = desc.chars().take(57).collect::(); + truncated.push_str("..."); + truncated + } else { + desc + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::migration::operations::SerializedColumn; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ryx_mig_{}_{}", name, std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::create_dir_all(&dir); + dir + } + + #[test] + fn test_write_and_load() { + let dir = temp_dir("write_load"); + let ops = vec![Operation::CreateTable { + table_name: "authors".into(), + columns: vec![SerializedColumn { + name: "id".into(), + db_type: "INTEGER".into(), + primary_key: true, + unique: true, + nullable: false, + default: None, + }], + model_name: Some("test::Author".into()), + database: Some("default".into()), + schema: String::new(), + }]; + + let path = write_migration_file(&ops, &dir).unwrap(); + assert!(path.exists()); + assert!(path.file_name().unwrap().to_str().unwrap().starts_with("0001")); + + let loaded = load_migration_file(&path).unwrap(); + assert_eq!(loaded.operations.len(), 1); + assert!(loaded.dependencies.is_empty()); + + // Cleanup + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn test_discover_sorts_globally() { + let dir = temp_dir("discover"); + let sub = dir.join("sub"); + std::fs::create_dir_all(&sub).unwrap(); + + let ops = vec![Operation::CreateTable { + table_name: "t".into(), + columns: vec![], + model_name: None, + database: None, + schema: String::new(), + }]; + + // File in subdirectory first (by numeric prefix) + let _ = write_migration_file(&ops, &sub).unwrap(); + // Write another with the incremented number + let _ = write_migration_file(&ops, &dir).unwrap(); + + let files = discover_migration_files(&dir); + assert_eq!(files.len(), 2); + assert!(files[0].to_str().unwrap().contains("0001")); + assert!(files[1].to_str().unwrap().contains("0002")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn test_slug_from_operations() { + let ops = vec![Operation::CreateTable { + table_name: "posts".into(), + columns: vec![], + model_name: None, + database: None, + schema: String::new(), + }]; + assert_eq!(slug_from_operations(&ops), "create_posts"); + } +} diff --git a/ryx-rs/src/migration/operations.rs b/ryx-rs/src/migration/operations.rs new file mode 100644 index 0000000..9c0f294 --- /dev/null +++ b/ryx-rs/src/migration/operations.rs @@ -0,0 +1,281 @@ +use serde::{Deserialize, Serialize}; + +/// A single migration operation, serializable to/from YAML migration files. +/// +/// Each variant stores the target table name and an optional model/database +/// reference used by the runner for per-alias routing. +/// +/// ```yaml +/// - type: CreateTable +/// table_name: authors +/// model_name: myapp::Author +/// database: blog +/// columns: +/// - { name: id, db_type: INTEGER, pk: true, unique: true } +/// - { name: name, db_type: VARCHAR(100), nullable: false } +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type")] +pub enum Operation { + /// Create a new table with the given columns. + #[serde(rename = "CreateTable")] + CreateTable { + table_name: String, + columns: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + model_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + database: Option, + /// Database schema (PostgreSQL). Empty = default. + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Add a column to an existing table. + #[serde(rename = "AddField")] + AddField { + table_name: String, + column: SerializedColumn, + #[serde(skip_serializing_if = "Option::is_none")] + model_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + database: Option, + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Drop a column. Destructive — not auto-generated. + #[serde(rename = "RemoveField")] + RemoveField { + table_name: String, + column_name: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Alter a column definition (type, nullability, etc.). + #[serde(rename = "AlterField")] + AlterField { + table_name: String, + old_column: SerializedColumn, + new_column: SerializedColumn, + #[serde(skip_serializing_if = "Option::is_none")] + model_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + database: Option, + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Create an index on one or more columns. + #[serde(rename = "CreateIndex")] + CreateIndex { + table_name: String, + index_name: String, + fields: Vec, + unique: bool, + #[serde(skip_serializing_if = "Option::is_none")] + model_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + database: Option, + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Drop an index. + #[serde(rename = "DeleteIndex")] + DeleteIndex { + table_name: String, + index_name: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + schema: String, + }, + /// Raw SQL to run forward. ``reverse_sql`` is for rollback. + /// Applies to all aliases (no table/model routing). + #[serde(rename = "RunSQL")] + RunSQL { + sql: String, + #[serde(skip_serializing_if = "Option::is_none")] + reverse_sql: Option, + }, + /// Create a new database schema (e.g. ``CREATE SCHEMA IF NOT EXISTS "tenant1"``). + /// Only relevant for PostgreSQL. + #[serde(rename = "CreateSchema")] + CreateSchema { + schema_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + database: Option, + }, +} + +/// A column definition suitable for YAML serialization. +/// +/// Mirrors ``ColumnState`` but uses flat fields for readability. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SerializedColumn { + pub name: String, + pub db_type: String, + #[serde(default)] + pub nullable: bool, + #[serde(default, rename = "pk")] + pub primary_key: bool, + #[serde(default)] + pub unique: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, +} + +impl Operation { + /// The target database alias for this operation, if set. + /// + /// Operations without a database are considered relevant for **all** + /// aliases (legacy / raw-SQL fallback). + pub fn database(&self) -> Option<&str> { + match self { + Operation::CreateTable { database, .. } + | Operation::AddField { database, .. } + | Operation::AlterField { database, .. } + | Operation::CreateIndex { database, .. } + | Operation::CreateSchema { database, .. } => database.as_deref(), + Operation::RemoveField { .. } + | Operation::DeleteIndex { .. } + | Operation::RunSQL { .. } => None, + } + } + + /// The model name this operation targets. + pub fn model_name(&self) -> Option<&str> { + match self { + Operation::CreateTable { model_name, .. } + | Operation::AddField { model_name, .. } + | Operation::AlterField { model_name, .. } + | Operation::CreateIndex { model_name, .. } => model_name.as_deref(), + Operation::RemoveField { .. } + | Operation::DeleteIndex { .. } + | Operation::RunSQL { .. } + | Operation::CreateSchema { .. } => None, + } + } + + /// The table name this operation acts on. + pub fn table_name(&self) -> Option<&str> { + match self { + Operation::CreateTable { table_name, .. } + | Operation::AddField { table_name, .. } + | Operation::RemoveField { table_name, .. } + | Operation::AlterField { table_name, .. } + | Operation::CreateIndex { table_name, .. } + | Operation::DeleteIndex { table_name, .. } => Some(table_name.as_str()), + Operation::RunSQL { .. } | Operation::CreateSchema { .. } => None, + } + } + + /// The target schema for this operation, if set. + /// + /// An empty string means the default schema for the backend (no qualification). + pub fn schema(&self) -> &str { + match self { + Operation::CreateTable { schema, .. } + | Operation::AddField { schema, .. } + | Operation::RemoveField { schema, .. } + | Operation::AlterField { schema, .. } + | Operation::CreateIndex { schema, .. } + | Operation::DeleteIndex { schema, .. } => schema.as_str(), + Operation::CreateSchema { schema_name, .. } => schema_name.as_str(), + Operation::RunSQL { .. } => "", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn col(name: &str) -> SerializedColumn { + SerializedColumn { + name: name.into(), + db_type: "TEXT".into(), + nullable: true, + primary_key: false, + unique: false, + default: None, + } + } + + #[test] + fn test_create_schema_operation() { + let op = Operation::CreateSchema { + schema_name: "tenant1".into(), + database: None, + }; + assert_eq!(op.table_name(), None); + assert_eq!(op.schema(), "tenant1"); + } + + #[test] + fn test_create_table_operation_schema() { + let op = Operation::CreateTable { + table_name: "posts".into(), + schema: "tenant1".into(), + columns: vec![], + model_name: None, + database: None, + }; + assert_eq!(op.schema(), "tenant1"); + assert_eq!(op.table_name(), Some("posts")); + } + + #[test] + fn test_add_field_operation_schema() { + let op = Operation::AddField { + table_name: "posts".into(), + schema: "tenant1".into(), + column: col("title"), + model_name: None, + database: None, + }; + assert_eq!(op.schema(), "tenant1"); + } + + #[test] + fn test_alter_field_operation_schema() { + let op = Operation::AlterField { + table_name: "posts".into(), + schema: "tenant1".into(), + old_column: col("title"), + new_column: col("title"), + model_name: None, + database: None, + }; + assert_eq!(op.schema(), "tenant1"); + } + + #[test] + fn test_create_index_operation_schema() { + let op = Operation::CreateIndex { + table_name: "posts".into(), + schema: "tenant1".into(), + index_name: "idx_title".into(), + fields: vec!["title".into()], + unique: false, + model_name: None, + database: None, + }; + assert_eq!(op.schema(), "tenant1"); + } + + #[test] + fn test_delete_index_operation_schema() { + let op = Operation::DeleteIndex { + table_name: "posts".into(), + schema: "tenant1".into(), + index_name: "idx_title".into(), + }; + assert_eq!(op.schema(), "tenant1"); + } + + #[test] + fn test_remove_field_operation_schema() { + let op = Operation::RemoveField { + table_name: "posts".into(), + schema: "tenant1".into(), + column_name: "title".into(), + }; + assert_eq!(op.schema(), "tenant1"); + } +} diff --git a/ryx-rs/src/migration/runner.rs b/ryx-rs/src/migration/runner.rs new file mode 100644 index 0000000..f4b4d36 --- /dev/null +++ b/ryx-rs/src/migration/runner.rs @@ -0,0 +1,573 @@ +use std::collections::HashSet; +use std::path::PathBuf; + +use ryx_common::RyxResult; + +use crate::migration::autodetect::{Autodetector, ModelEntry}; +use crate::migration::ddl::DDLGenerator; +use crate::migration::files::{discover_migration_files, load_migration_file}; +use crate::migration::operations::Operation; +use crate::migration::{ + diff_states, generate_ddl, get_text, introspect_schema, ColumnState, SchemaState, TableState, + MIGRATIONS_TABLE, +}; + +/// File-based migration runner with per-alias tracking and live fallback. +/// +/// ```ignore +/// use ryx_rs::migration::FileRunner; +/// +/// FileRunner::new() +/// .model::() +/// .model::() +/// .migrations_dir("migrations/") +/// .run().await?; +/// ``` +pub struct FileRunner { + models: Vec TableState>, + model_entries: Vec, + db_alias: Option, + /// Database schema for PostgreSQL multi-schema support (empty = default). + schema: String, + migrations_dir: String, + dry_run: bool, + no_interactive: bool, + /// When true, skip file discovery entirely and run a live auto-diff. + live: bool, +} + +impl FileRunner { + pub fn new() -> Self { + Self { + models: vec![], + model_entries: vec![], + db_alias: None, + schema: String::new(), + migrations_dir: "migrations".to_string(), + dry_run: false, + no_interactive: false, + live: false, + } + } + + pub fn db(mut self, alias: &str) -> Self { + self.db_alias = Some(alias.to_string()); + self + } + + /// Set the database schema for PostgreSQL multi-schema support. + /// + /// When set, all introspection, DDL, and operations will be scoped + /// to this schema (e.g. ``CREATE TABLE "tenant1"."posts"``). + /// Leave empty for default schema (no qualification). + pub fn schema(mut self, schema: &str) -> Self { + self.schema = schema.to_string(); + self + } + + pub fn migrations_dir(mut self, dir: &str) -> Self { + self.migrations_dir = dir.to_string(); + self + } + + pub fn dry_run(mut self, dry: bool) -> Self { + self.dry_run = dry; + self + } + + pub fn no_interactive(mut self, no: bool) -> Self { + self.no_interactive = no; + self + } + + /// Skip file discovery — always run a live auto-diff against the DB. + pub fn live(mut self, live: bool) -> Self { + self.live = live; + self + } + + pub fn model(mut self) -> Self { + let info_fn = || -> TableState { + let meta = M::field_meta(); + let columns: Vec = meta.iter().map(|m| m.into()).collect(); + TableState { + name: M::table_name().to_string(), + schema: String::new(), + columns, + } + }; + self.models.push(info_fn); + let mut entry = ModelEntry::from_model::(); + if !self.schema.is_empty() { + entry = entry.with_schema(&self.schema); + } + self.model_entries.push(entry); + self + } + + fn build_target(&self) -> SchemaState { + let mut tables: Vec = self.models.iter().map(|f| f()).collect(); + if !self.schema.is_empty() { + for table in &mut tables { + table.schema = self.schema.clone(); + } + } + SchemaState { tables } + } + + // ── Public API ────────────────────────────────────── + + pub async fn run(self) -> RyxResult> { + let alias = self.db_alias.clone().unwrap_or_else(|| "default".to_string()); + self.ensure_tracking_table(&alias).await?; + + if self.live { + return self.run_live(&alias).await; + } + + let dir = std::path::Path::new(&self.migrations_dir); + let files = discover_migration_files(dir); + + if files.is_empty() { + self.handle_no_files(&alias).await + } else { + self.apply_files(&alias, &files).await + } + } + + pub async fn plan(self) -> RyxResult> { + let alias = self.db_alias.clone().unwrap_or_else(|| "default".to_string()); + + if self.live { + return self.plan_live(&alias).await; + } + + let dir = std::path::Path::new(&self.migrations_dir); + let files = discover_migration_files(dir); + + if files.is_empty() { + self.plan_live(&alias).await + } else { + self.plan_files(&alias, &files).await + } + } + + async fn run_live(&self, alias: &str) -> RyxResult> { + let target = self.build_target(); + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let current = introspect_schema(pool.as_ref(), backend_type, &self.schema).await?; + let changes = diff_states(¤t, &target); + if changes.is_empty() { + return Ok(vec![]); + } + let ddl = if self.schema.is_empty() { + generate_ddl(&changes, backend_type) + } else { + DDLGenerator::new(backend_type) + .in_schema(&self.schema) + .generate(&changes) + }; + let mut results = Vec::new(); + for stmt in &ddl { + if self.dry_run { + println!("{stmt}"); + } else { + let _ = pool.fetch_raw(stmt.clone(), None).await?; + } + results.push(stmt.clone()); + } + Ok(results) + } + + async fn plan_live(&self, alias: &str) -> RyxResult> { + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let current = introspect_schema(pool.as_ref(), backend_type, &self.schema).await?; + let target = self.build_target(); + let changes = diff_states(¤t, &target); + let ddl = if self.schema.is_empty() { + generate_ddl(&changes, backend_type) + } else { + DDLGenerator::new(backend_type) + .in_schema(&self.schema) + .generate(&changes) + }; + Ok(ddl) + } + + // ── File pipeline ─────────────────────────────────── + + async fn apply_files( + &self, + alias: &str, + files: &[PathBuf], + ) -> RyxResult> { + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let base_ddl = if self.schema.is_empty() { + DDLGenerator::new(backend_type) + } else { + DDLGenerator::new(backend_type).in_schema(&self.schema) + }; + let applied = self.get_applied_migrations(&pool, alias).await?; + let mut results = Vec::new(); + + for path in files { + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + let key = format!("{alias}|{stem}"); + if applied.contains(&key) { + continue; + } + + let mf = match load_migration_file(path) { + Ok(m) => m, + Err(e) => { + eprintln!("[ryx] Skipping {}: {e}", path.display()); + continue; + } + }; + + for op in &mf.operations { + if !operation_is_relevant(op, alias) { + continue; + } + let mut ddl = base_ddl.clone(); + if !op.schema().is_empty() { + ddl.schema = op.schema().to_string(); + } + for sql in operation_to_sql(&ddl, op) { + if self.dry_run { + println!("{sql}"); + } else { + let _ = pool.fetch_raw(sql.clone(), None).await?; + } + results.push(sql); + } + } + + if !self.dry_run { + self.record_migration(&pool, &key).await?; + } + } + Ok(results) + } + + async fn plan_files( + &self, + alias: &str, + files: &[PathBuf], + ) -> RyxResult> { + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let base_ddl = if self.schema.is_empty() { + DDLGenerator::new(backend_type) + } else { + DDLGenerator::new(backend_type).in_schema(&self.schema) + }; + let applied = self.get_applied_migrations(&pool, alias).await?; + let mut results = Vec::new(); + + for path in files { + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + let key = format!("{alias}|{stem}"); + if applied.contains(&key) { + continue; + } + if let Ok(mf) = load_migration_file(path) { + for op in &mf.operations { + if operation_is_relevant(op, alias) { + let mut ddl = base_ddl.clone(); + if !op.schema().is_empty() { + ddl.schema = op.schema().to_string(); + } + results.extend(operation_to_sql(&ddl, op)); + } + } + } + } + Ok(results) + } + + // ── Tracking table ────────────────────────────────── + + async fn ensure_tracking_table(&self, alias: &str) -> RyxResult<()> { + let pool = ryx_backend::pool::get(Some(alias))?; + let sql = format!( + "CREATE TABLE IF NOT EXISTS \"{t}\" (\ + id INTEGER PRIMARY KEY,\ + name VARCHAR(255) NOT NULL UNIQUE,\ + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP\ + )", + t = MIGRATIONS_TABLE + ); + let _ = pool.fetch_raw(sql, None).await?; + Ok(()) + } + + async fn get_applied_migrations( + &self, + pool: &std::sync::Arc, + alias: &str, + ) -> RyxResult> { + let sql = format!("SELECT name FROM \"{}\"", MIGRATIONS_TABLE); + let rows = pool.fetch_raw(sql, None).await?; + let mut set = HashSet::new(); + + for row in &rows { + if let Some(name) = get_text(row, "name") { + if name.contains('|') { + set.insert(name); + } else { + set.insert(format!("{alias}|{name}")); + } + } + } + Ok(set) + } + + async fn record_migration( + &self, + pool: &std::sync::Arc, + key: &str, + ) -> RyxResult<()> { + let sql = format!( + "INSERT OR REPLACE INTO \"{}\" (name) VALUES ('{key}')", + MIGRATIONS_TABLE + ); + let _ = pool.fetch_raw(sql, None).await?; + Ok(()) + } + + // ── No-files fallback ─────────────────────────────── + + async fn handle_no_files(&self, alias: &str) -> RyxResult> { + let target = self.build_target(); + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let current = introspect_schema(pool.as_ref(), backend_type, &self.schema).await?; + let changes = diff_states(¤t, &target); + + if changes.is_empty() { + return Ok(vec![]); + } + if self.no_interactive { + eprintln!( + "[ryx] No migration files exist and --no-interactive is set.\n\ + Run 'ryx makemigrations' first." + ); + return Ok(vec![]); + } + self.interactive_menu(alias, &changes).await + } + + async fn interactive_menu( + &self, + alias: &str, + changes: &[crate::migration::SchemaChange], + ) -> RyxResult> { + println!(); + println!( + "[ryx] No migration files exist for database '{alias}'" + ); + println!(" {} model(s) are not yet tracked.", self.models.len()); + println!(); + println!(" [L]ive DDL — apply changes directly (development only)"); + println!(" [A]uto-generate migration files, then migrate"); + println!(" [M]anual — run 'ryx makemigrations' first"); + println!(" [S]kip this database for now"); + println!(); + + let mut choice = String::new(); + print!("[ryx] Choice (L/A/M/S) [S]: "); + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stdin().read_line(&mut choice); + let choice = choice.trim().to_uppercase(); + + match choice.as_str() { + "L" => { + println!( + "[ryx] Applying {} live change(s) to {alias}", + changes.len() + ); + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let ddl = generate_ddl(changes, backend_type); + let mut results = Vec::new(); + for stmt in &ddl { + if self.dry_run { + println!("{stmt}"); + } else { + let _ = pool.fetch_raw(stmt.clone(), None).await?; + } + results.push(stmt.clone()); + } + Ok(results) + } + "A" => { + let detector = + Autodetector::new(self.model_entries.clone(), &self.migrations_dir); + let ops = detector.detect(); + if ops.is_empty() { + println!("[ryx] No changes detected."); + return Ok(vec![]); + } + let path = detector + .write_migration(&ops) + .map_err(|e| ryx_common::RyxError::Internal(e))?; + println!("[ryx] Created migration: {}", path.display()); + + let pool = ryx_backend::pool::get(Some(alias))?; + let backend_type = ryx_backend::pool::get_backend(Some(alias))?; + let ddl = DDLGenerator::new(backend_type); + let mut results = Vec::new(); + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + let key = format!("{alias}|{stem}"); + + for op in &ops { + for sql in operation_to_sql(&ddl, op) { + if self.dry_run { + println!("{sql}"); + } else { + let _ = pool.fetch_raw(sql.clone(), None).await?; + } + results.push(sql); + } + } + if !self.dry_run { + self.record_migration(&pool, &key).await?; + } + println!("[ryx] applied"); + Ok(results) + } + "M" => { + println!("[ryx] Run: ryx makemigrations --models "); + println!("[ryx] Then run 'ryx migrate' again."); + Ok(vec![]) + } + _ => { + println!("[ryx] Skipping database '{alias}'."); + Ok(vec![]) + } + } + } +} + +impl Default for FileRunner { + fn default() -> Self { + Self::new() + } +} + +// ── Helpers ──────────────────────────────────────────── + +/// Check whether an operation should run for a given database alias. +pub fn operation_is_relevant(op: &Operation, alias: &str) -> bool { + match op.database() { + Some(db) => db == alias, + None => true, + } +} + +/// Convert an ``Operation`` to backend-aware DDL statements. +pub fn operation_to_sql(ddl: &DDLGenerator, op: &Operation) -> Vec { + match op { + Operation::CreateSchema { schema_name, .. } => { + let sql = ddl.create_schema(schema_name); + if sql.is_empty() { + vec![] + } else { + vec![sql] + } + } + Operation::CreateTable { + table_name, columns, .. + } => { + let cols: Vec = columns + .iter() + .map(|c| ColumnState { + name: c.name.clone(), + db_type: c.db_type.clone(), + nullable: c.nullable, + primary_key: c.primary_key, + unique: c.unique, + default: c.default.clone(), + }) + .collect(); + vec![ddl.create_table(&TableState { + name: table_name.clone(), + schema: ddl.schema.clone(), + columns: cols, + })] + } + Operation::AddField { + table_name, column, .. + } => { + vec![ddl.add_column( + table_name, + &ColumnState { + name: column.name.clone(), + db_type: column.db_type.clone(), + nullable: column.nullable, + primary_key: column.primary_key, + unique: column.unique, + default: column.default.clone(), + }, + )] + } + Operation::RemoveField { + table_name, + column_name, + .. + } => vec![ddl.drop_column(table_name, column_name)], + Operation::AlterField { + table_name, + old_column, + new_column, + .. + } => { + ddl.alter_column( + table_name, + &ColumnState { + name: old_column.name.clone(), + db_type: old_column.db_type.clone(), + nullable: old_column.nullable, + primary_key: old_column.primary_key, + unique: old_column.unique, + default: old_column.default.clone(), + }, + &ColumnState { + name: new_column.name.clone(), + db_type: new_column.db_type.clone(), + nullable: new_column.nullable, + primary_key: new_column.primary_key, + unique: new_column.unique, + default: new_column.default.clone(), + }, + ) + } + Operation::CreateIndex { + table_name, + index_name, + fields, + unique, + .. + } => vec![ddl.create_index(table_name, index_name, fields, *unique)], + Operation::DeleteIndex { + table_name, + index_name, + .. + } => vec![ddl.drop_index(table_name, index_name)], + Operation::RunSQL { sql, .. } => vec![sql.clone()], + } +} diff --git a/ryx-rs/src/model.rs b/ryx-rs/src/model.rs index 3e78d1c..4fb2c12 100644 --- a/ryx-rs/src/model.rs +++ b/ryx-rs/src/model.rs @@ -24,6 +24,14 @@ pub trait Model: Send + Sync + 'static { /// Used by the migration system to compare the model's schema /// against the live database. fn field_meta() -> &'static [FieldMeta]; + /// Database alias this model belongs to — used by the migration system + /// to route operations to the correct database. + /// + /// Set via `#[database("blog")]` on the struct or `#[model(database = "blog")]`. + /// Defaults to `"default"`. + fn database() -> &'static str { + "default" + } } /// Metadata for a foreign-key relationship. diff --git a/ryx-rs/src/queryset.rs b/ryx-rs/src/queryset.rs index 43b5109..c07ae6d 100644 --- a/ryx-rs/src/queryset.rs +++ b/ryx-rs/src/queryset.rs @@ -69,6 +69,16 @@ impl QuerySet { self } + /// Set the database schema for this query (PostgreSQL multi-schema). + /// + /// ```ignore + /// Post::objects().schema("tenant1").all().await?; + /// ``` + pub fn schema(mut self, schema: &str) -> Self { + self.node = self.node.with_schema(schema); + self + } + // === FILTERS === /// Add a filter condition. diff --git a/ryx-rs/tests/full_pipeline_test.rs b/ryx-rs/tests/full_pipeline_test.rs index 85abdbc..c5d20f2 100644 --- a/ryx-rs/tests/full_pipeline_test.rs +++ b/ryx-rs/tests/full_pipeline_test.rs @@ -98,6 +98,7 @@ async fn full_pipeline_test() { // 1. Create tables via MigrationRunner MigrationRunner::new() + .live(true) .model::() .model::() .run() diff --git a/ryx-rs/tests/migration_test.rs b/ryx-rs/tests/migration_test.rs index 07d1eaa..0f08c4a 100644 --- a/ryx-rs/tests/migration_test.rs +++ b/ryx-rs/tests/migration_test.rs @@ -102,6 +102,8 @@ async fn sequential_migration_tests() { // Test 1: create table MigrationRunner::new() + .live(true) + .live(true) .model::() .run() .await @@ -114,6 +116,7 @@ async fn sequential_migration_tests() { // Test 2: add column MigrationRunner::new() + .live(true) .model::() .run() .await @@ -125,6 +128,7 @@ async fn sequential_migration_tests() { // Test 3: multiple models MigrationRunner::new() + .live(true) .model::() .run() .await @@ -133,6 +137,7 @@ async fn sequential_migration_tests() { // Test 4: idempotent MigrationRunner::new() + .live(true) .model::() .model::() .run() diff --git a/ryx-rs/tests/multi_schema_test.rs b/ryx-rs/tests/multi_schema_test.rs new file mode 100644 index 0000000..6eede34 --- /dev/null +++ b/ryx-rs/tests/multi_schema_test.rs @@ -0,0 +1,216 @@ +use std::collections::HashMap; + +use ryx_rs::migration::MigrationRunner; +use ryx_rs::model; +use ryx_rs::PoolConfig; + +// ── Model definitions ───────────────────────────────── + +#[model] +#[table("ms_posts")] +struct MsPost { + #[field(pk)] + id: i64, + title: String, +} + +#[model] +#[table("ms_authors")] +struct MsAuthor { + #[field(pk)] + id: i64, + name: String, +} + +// ── Helpers ──────────────────────────────────────────── + +fn pg_url() -> Option { + std::env::var("PG_TEST_URL").ok().or_else(|| { + Some("postgres://einswilli@localhost/ryx_integration_test".into()) + }) +} + +async fn init_pg_pool(url: &str) { + let mut urls = HashMap::new(); + urls.insert("default".into(), url.into()); + ryx_backend::pool::initialize(urls, PoolConfig { + max_connections: 2, + min_connections: 1, + ..Default::default() + }) + .await + .expect("Failed to init PG pool"); +} + +async fn table_exists_in_schema(table: &str, schema: &str) -> bool { + let backend = ryx_backend::pool::get(None).unwrap(); + let rows = backend + .fetch_raw( + format!( + "SELECT table_name FROM information_schema.tables \ + WHERE table_schema = '{schema}' AND table_name = '{table}'" + ), + None, + ) + .await + .unwrap(); + !rows.is_empty() +} + +async fn cleanup_schema(schema: &str) { + let backend = ryx_backend::pool::get(None).unwrap(); + // Drop all tables in the schema first + let _ = backend + .fetch_raw( + format!( + "SELECT string_agg(format('DROP TABLE IF EXISTS %I.%I CASCADE', \ + table_schema, table_name), '; ') AS sql \ + FROM information_schema.tables \ + WHERE table_schema = '{schema}' AND table_type = 'BASE TABLE'" + ), + None, + ) + .await; + + let _ = backend + .fetch_raw(format!("DROP SCHEMA IF EXISTS \"{schema}\" CASCADE"), None) + .await; +} + +// ── Multi-schema integration tests ───────────────────── + +#[tokio::test] +async fn multi_schema_migration_pipeline() { + let url = match pg_url() { + Some(u) => u, + None => { + eprintln!("Skipping: PG_TEST_URL not set and default PG may not be available"); + return; + } + }; + + init_pg_pool(&url).await; + + // Clean up from previous runs (commented out for DB inspection) + // cleanup_schema("tenant1").await; + // cleanup_schema("tenant2").await; + + // 1. Run migration in tenant1 schema + MigrationRunner::new() + .live(true) + .schema("tenant1") + .model::() + .model::() + .run() + .await + .expect("tenant1 migration should succeed"); + + assert!( + table_exists_in_schema("ms_posts", "tenant1").await, + "ms_posts should exist in tenant1 schema" + ); + assert!( + table_exists_in_schema("ms_authors", "tenant1").await, + "ms_authors should exist in tenant1 schema" + ); + + // 2. Tables should NOT exist in public schema + assert!( + !table_exists_in_schema("ms_posts", "public").await, + "ms_posts should NOT exist in public schema" + ); + + // 3. Run migration in tenant2 schema (same models, different schema) + MigrationRunner::new() + .live(true) + .schema("tenant2") + .model::() + .model::() + .run() + .await + .expect("tenant2 migration should succeed"); + + assert!( + table_exists_in_schema("ms_posts", "tenant2").await, + "ms_posts should exist in tenant2 schema" + ); + assert!( + table_exists_in_schema("ms_authors", "tenant2").await, + "ms_authors should exist in tenant2 schema" + ); + + // 4. Tenant1 still has its tables + assert!( + table_exists_in_schema("ms_posts", "tenant1").await, + "tenant1.ms_posts should still exist" + ); + + // 5. Insert data into tenant1 via raw query + { + let backend = ryx_backend::pool::get(None).unwrap(); + backend + .fetch_raw( + "INSERT INTO \"tenant1\".\"ms_posts\" (title) VALUES ('Post 1'), ('Post 2')" + .into(), + None, + ) + .await + .expect("Insert into tenant1 should work"); + } + + // 6. Insert data into tenant2 (independent) + { + let backend = ryx_backend::pool::get(None).unwrap(); + backend + .fetch_raw( + "INSERT INTO \"tenant2\".\"ms_posts\" (title) VALUES ('Tenant 2 Post')".into(), + None, + ) + .await + .expect("Insert into tenant2 should work"); + } + + // 7. Verify count in tenant1 + { + let backend = ryx_backend::pool::get(None).unwrap(); + let rows = backend + .fetch_raw( + "SELECT count(*) AS cnt FROM \"tenant1\".\"ms_posts\"".into(), + None, + ) + .await + .unwrap(); + let count: i64 = rows[0] + .get("cnt") + .and_then(|v| match v { + ryx_rs::SqlValue::Int(n) => Some(*n), + _ => None, + }) + .unwrap_or(0); + assert_eq!(count, 2, "tenant1 should have 2 posts"); + } + + // 8. Verify count in tenant2 + { + let backend = ryx_backend::pool::get(None).unwrap(); + let rows = backend + .fetch_raw( + "SELECT count(*) AS cnt FROM \"tenant2\".\"ms_posts\"".into(), + None, + ) + .await + .unwrap(); + let count: i64 = rows[0] + .get("cnt") + .and_then(|v| match v { + ryx_rs::SqlValue::Int(n) => Some(*n), + _ => None, + }) + .unwrap_or(0); + assert_eq!(count, 1, "tenant2 should have 1 post"); + } + + // Cleanup (skipped for DB inspection) + // cleanup_schema("tenant1").await; + // cleanup_schema("tenant2").await; +} diff --git a/ryx-rs/tests/relation_test.rs b/ryx-rs/tests/relation_test.rs index dcbb707..cca4909 100644 --- a/ryx-rs/tests/relation_test.rs +++ b/ryx-rs/tests/relation_test.rs @@ -72,6 +72,7 @@ async fn test_select_related_basic() { // Create tables MigrationRunner::new() + .live(true) .model::() .model::() .run()