diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1ad1812 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# CipherStash Go Encryption SDK — example environment configuration. +# +# Copy this file to .env and fill in the values from your CipherStash workspace +# (`stash setup` writes them to cipherstash.toml / cipherstash.secret.toml, or +# you can set them here). Do not commit real credentials. + +# Workspace CRN — identifies your workspace. Required for OIDC federation +# (WithOIDCFederation) when it is not supplied via WithCredentials. +export CS_WORKSPACE_CRN=crn:.: + +# Access key — authenticates the client to CipherStash. +export CS_CLIENT_ACCESS_KEY= + +# Client ID and client key — the encryption key material. These are always +# required to encrypt and decrypt, regardless of the authentication strategy. +export CS_CLIENT_ID= +export CS_CLIENT_KEY= diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7de7993..6125b45 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,6 +84,12 @@ jobs: rust_target: x86_64-unknown-linux-gnu libc: glibc + # Linux arm64 glibc (for Debian/Ubuntu/CentOS) + - platform: linux-arm64-gnu + os: ubuntu-22.04-arm + rust_target: aarch64-unknown-linux-gnu + libc: glibc + # Linux arm64 musl (for Alpine and static linking) - platform: linux-arm64-musl os: ubuntu-22.04-arm @@ -126,7 +132,7 @@ jobs: if: contains(matrix.rust_target, 'musl') && startsWith(matrix.os, 'ubuntu') run: | mkdir -p .cargo - echo '[target.x86_64-unknown-linux-musl]' >> .cargo/config.toml + echo '[target.${{ matrix.rust_target }}]' >> .cargo/config.toml echo 'rustflags = ["-C", "target-feature=+crt-static"]' >> .cargo/config.toml - name: Build library @@ -165,6 +171,39 @@ jobs: path: pkg/protect/libprotect_ffi_*.a retention-days: 30 + # Run Go unit tests against the freshly built native libraries + test-go: + name: Go Tests (${{ matrix.platform }}) + needs: [build-native-libraries] + if: always() && needs.build-native-libraries.result == 'success' + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + os: macos-14 + - platform: linux-x64-gnu + os: ubuntu-22.04 + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Download library artifact + uses: actions/download-artifact@v4 + with: + name: library-${{ matrix.platform }} + path: pkg/protect + + - name: Run Go tests + run: go test ./pkg/protect/... + # Commit all generated libraries at once commit-artifacts: name: Commit All Generated Libraries diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0051e8e..631e815 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,124 +1,95 @@ # Development Guide -This repo is a combination of the Protect.go module and the Protect.go FFI for the Rust C library which is used to create bindings for the `cipherstash-client` crate. +This repo combines the CipherStash Go Encryption SDK with the Rust C FFI library +that wraps the `cipherstash-client` crate. ## Architecture The project consists of: -1. **Rust C Library** (`crates/protect-ffi-c/`) - A Rust library that exports C-compatible functions -2. **Go Package** (`pkg/protect/`) - Go bindings that wrap the C functions -3. **Examples** (`examples/`) - Usage examples showing how to use the library +1. **Rust C library** (`crates/protect-ffi-c/`) — a Rust crate that exports + C-compatible functions and generates the `protect_ffi.h` header. +2. **Go package** (`pkg/protect/`) — Go bindings that link the compiled static + library via cgo and expose an idiomatic Go API. +3. **Examples** (`examples/`) — runnable usage examples. ``` protectgo/ ├── crates/ │ └── protect-ffi-c/ # Rust C FFI library -├── pkg/protect/ # Go package +├── pkg/protect/ # Go package + precompiled static libraries ├── examples/ # Usage examples ``` -## Building +## Prerequisites -### Prerequisites +- Go (see the version in `go.mod`) +- A C toolchain (cgo is required) +- Rust (stable) — only needed if you are changing the Rust FFI layer -- [mise](https://mise.jdx.dev/) (handles Go and Rust versions automatically) -- CipherStash credentials and configuration +## How the native library is built -**Note**: mise will automatically install and manage the correct versions of Go (1.24.4) and Rust (nightly) for this project. +The Go package links a precompiled static library per platform, checked in under +`pkg/protect/` (for example `libprotect_ffi_darwin_arm64.a`). These are produced +in CI (see `.github/workflows/build.yml`) and committed back to the repo, so day +-to-day Go development does not require a Rust toolchain. -### Build Steps +To rebuild the native library locally after changing the Rust crate: -1. **Install mise** (if not already installed): - ```bash - # macOS - brew install mise - - # Linux - curl https://mise.run | sh - ``` +```bash +# Build for your host target +cargo build --release -2. **Install dependencies and tools**: - ```bash - mise run install-deps - ``` +# Copy the resulting archive to the matching platform filename, e.g. on Apple +# Silicon: +cp target/release/libprotect_ffi.a pkg/protect/libprotect_ffi_darwin_arm64.a +``` -3. **Build the project**: - ```bash - mise run build - ``` +The header `pkg/protect/protect_ffi.h` is regenerated by the crate's build +script (cbindgen) as part of `cargo build`. -4. **Run tests**: - ```bash - mise run test - ``` +## Building and testing the Go package -5. **Build and run example**: - ```bash - mise run example - ./bin/example - ``` +```bash +# Vet and build (links the platform static library via cgo) +go vet ./... +go build ./... -## Development +# Run the unit tests +go test ./... -### Project Structure +# Alpine Linux / musl targets +go test -tags=musl ./... +# Format +gofmt -w . ``` -protectgo/ -├── Cargo.toml # Rust workspace -├── go.mod # Go module -├── mise.toml # Task automation and tool management -├── Makefile # Legacy build automation (deprecated) -├── crates/ -│ └── protect-ffi-c/ # Rust C FFI library -│ ├── Cargo.toml -│ ├── build.rs # Build script for header generation -│ ├── cbindgen.toml # Header generation config -│ └── src/ -│ ├── lib.rs # Main FFI functions -│ └── encrypt_config.rs -├── pkg/ -│ └── protect/ # Go package -│ └── protect.go # Go bindings -└── examples/ - └── basic_usage.go # Usage example -``` - -### Building from Source - -1. Clone the repository -2. Install mise (see Build Steps above) -3. Run `mise run install-deps` to install dependencies and tools -4. Run `mise run build` to build the library -5. Run `mise run test` to run tests -**Available tasks**: Run `mise tasks` to see all available tasks including formatting, linting, and cleanup commands. +## Running the example -### Memory Management - -The Go bindings automatically handle memory management: -- C strings are automatically freed after use -- Client resources must be explicitly freed with `client.Free()` -- All returned data is copied to Go-managed memory +```bash +go run examples/basic_usage.go -## Testing +# Alpine Linux / musl +go run -tags=musl examples/basic_usage.go +``` -Run the test suite with: +You will need CipherStash credentials in the environment (see the README's +authentication section) for the example to talk to the service. -```bash -mise run test -``` +## Memory management -Additional development tasks: +The Go bindings handle memory management for you: -```bash -mise run fmt # Format code (Rust + Go) -mise run check # Run linting and quality checks -mise run clean # Clean build artifacts -``` +- C strings passed into the FFI are freed after each call. +- All data returned from the FFI is copied into Go-managed memory before the C + allocation is freed. +- Client resources are released with `client.Close()` — `Client` implements + `io.Closer`, so `defer client.Close()` is the idiomatic pattern. ## Support For support and questions: + - GitHub Issues: [protectgo/issues](https://github.com/cipherstash/protectgo/issues) -- CipherStash Documentation: [docs.cipherstash.com](https://docs.cipherstash.com) \ No newline at end of file +- CipherStash Documentation: [docs.cipherstash.com](https://docs.cipherstash.com) diff --git a/README.md b/README.md index 4bfb8ca..96409aa 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,30 @@ -> [!WARNING] +> [!WARNING] > This is a work in progress. > The package is not yet available on pkg.go.dev. The CipherStash Go Encryption SDK encrypts, decrypts, and searches encrypted data. Every value you encrypt has a unique key, made possible by CipherStash [ZeroKMS](https://cipherstash.com/products/zerokms)'s bulk key operations, backed by a root key in [AWS KMS](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html). The encrypted data is stored as a JSON payload in any database that supports JSONB. -> [!IMPORTANT] +> [!IMPORTANT] > Searching, sorting, and filtering on encrypted data requires PostgreSQL. +## Contents + +- [Quick start](#quick-start) +- [Installing](#installing) +- [Credentials and authentication](#credentials-and-authentication) +- [Defining your schema](#defining-your-schema) +- [Creating a client](#creating-a-client) +- [Encrypting and decrypting](#encrypting-and-decrypting) +- [Querying encrypted data](#querying-encrypted-data) +- [Identity-aware encryption](#identity-aware-encryption) +- [PostgreSQL setup](#postgresql-setup) +- [Error handling](#error-handling) +- [API reference](#api-reference) +- [Prebuilt libraries](#prebuilt-libraries) + ## Quick start ```go @@ -51,7 +66,7 @@ type User struct { ID int `json:"id"` Email string `json:"email" cs:"email,unique(downcase),match"` Name string `json:"name" cs:"name,match"` - Age int `json:"age" cs:"age,cast=number,ore"` + Age int `json:"age" cs:"age,ore"` Role string `json:"role"` } @@ -59,7 +74,10 @@ func main() { ctx := context.Background() // Build schema from struct tags - users, _ := protect.TableSchema("users", User{}) + users, err := protect.TableSchema("users", User{}) + if err != nil { + log.Fatal(err) + } // Create client — credentials from env vars or config files client, err := protect.NewClient(ctx, protect.WithSchemas(users)) @@ -77,7 +95,9 @@ func main() { // Decrypt back to struct var decrypted User - client.DecryptModel(ctx, users, encrypted, &decrypted) + if err := client.DecryptModel(ctx, users, encrypted, &decrypted); err != nil { + log.Fatal(err) + } log.Printf("%s <%s>", decrypted.Name, decrypted.Email) } ``` @@ -88,7 +108,13 @@ func main() { go get github.com/cipherstash/protectgo/pkg/protect ``` -### CipherStash CLI +The SDK links a precompiled native library via cgo — no Rust toolchain is +required, but `CGO_ENABLED=1` (the default) and a C toolchain are. See +[Prebuilt libraries](#prebuilt-libraries) for the supported platforms. + +## Credentials and authentication + +### Setting up credentials ```bash # macOS @@ -100,32 +126,100 @@ stash setup This creates `cipherstash.toml` and `cipherstash.secret.toml` in your project. -> [!WARNING] +> [!WARNING] > Don't commit `cipherstash.secret.toml` to git. -You can also use environment variables: +You can also use environment variables (see `.env.example`): | Variable | Description | |---|---| -| `CS_WORKSPACE_CRN` | Workspace CRN | -| `CS_CLIENT_ACCESS_KEY` | Access key | -| `CS_CLIENT_ID` | Client ID | +| `CS_WORKSPACE_CRN` | Workspace CRN, e.g. `crn:ap-southeast-2.aws:WORKSPACEID` | +| `CS_CLIENT_ACCESS_KEY` | Access key (`CS_ACCESS_KEY` is also accepted) | +| `CS_CLIENT_ID` | Client ID (used only when `CS_CLIENT_KEY` is also set) | | `CS_CLIENT_KEY` | Client key | +Or pass everything explicitly: + +```go +client, err := protect.NewClient(ctx, + protect.WithSchemas(users), + protect.WithCredentials(workspaceCRN, accessKey, clientID, clientKey), +) +``` + +Two independent things are being configured here: + +- **Authentication** — how the client proves who it is to CipherStash. By + default this is the access key; alternatively use + [OIDC federation](#per-user-identity-with-oidc-federation) or a custom + token provider. +- **Key material** — the client ID and client key pair. This is **always + required** to encrypt and decrypt values, regardless of which + authentication strategy you use. + +### Per-user identity with OIDC federation + +`WithOIDCFederation` makes every encryption and decryption identity-aware at +the client level — no per-operation configuration required. Supply a function +that returns a fresh OIDC access token (a JWT) from your application's +identity provider (Clerk, Auth0, Supabase, and similar). The client exchanges +that token for a short-lived CipherStash service token, verifies it belongs to +your workspace, and caches it until expiry — your function is called again +only when re-federation is needed. + +```go +client, err := protect.NewClient(ctx, + protect.WithSchemas(users), + protect.WithCredentials(crn, "", clientID, clientKey), // no access key needed + protect.WithOIDCFederation(func(ctx context.Context) (string, error) { + return identityProvider.AccessToken(ctx) // your app's IdP JWT + }), +) +``` + +A workspace CRN is required — supply it via `WithCredentials` or +`CS_WORKSPACE_CRN`. `NewClient` returns an error wrapping `ErrAuthStrategy` if +neither is present. + +Because federation happens per client, the natural pattern for per-user +isolation is one client per user session (or per request), each federating +that user's JWT. Key operations are then attributed to that user in +CipherStash audit logs and governed by that user's policy. + +### Custom token provider + +For advanced setups, `WithTokenProvider` supplies a CipherStash service token +directly. Your function is called on every keyservice request — caching and +refresh are your responsibility: + +```go +client, err := protect.NewClient(ctx, + protect.WithSchemas(users), + protect.WithTokenProvider(func(ctx context.Context) (string, error) { + return myTokenCache.Current(ctx) + }), +) +``` + +`WithOIDCFederation` and `WithTokenProvider` are mutually exclusive. + ## Defining your schema ### Struct tags -The `cs` tag defines the encryption schema directly on your Go structs: +The `cs` tag defines the encryption schema directly on your Go structs. The +first tag element is the column name; the rest are directives: ```go type User struct { - ID int `json:"id"` // not encrypted - Email string `json:"email" cs:"email,unique(downcase),match"` // exact match + full-text - Name string `json:"name" cs:"name,match"` // full-text search - Age int `json:"age" cs:"age,cast=number,ore"` // range queries - Metadata any `json:"metadata" cs:"metadata,ste_vec(prefix=u/m)"` // JSON queries - Role string `json:"role"` // not encrypted + ID int `json:"id"` // not encrypted + Email string `json:"email" cs:"email,unique(downcase),match"` // exact match + full-text + Name string `json:"name" cs:"name,match"` // full-text search + Age int `json:"age" cs:"age,ore"` // range queries + Salary float64 `json:"salary" cs:"salary,cast=decimal"` // exact decimal storage + Started time.Time `json:"started" cs:"started,ore"` // sortable timestamp + Metadata any `json:"metadata" cs:"metadata,ste_vec(prefix=users/metadata)"` // JSON queries + Role string `json:"role"` // not encrypted } users, err := protect.TableSchema("users", User{}) @@ -133,36 +227,46 @@ users, err := protect.TableSchema("users", User{}) #### Index directives -| Directive | Description | +| Directive | Enables | |---|---| -| `unique` | Exact match queries (`WHERE email = ?`) | +| `unique` | Exact-match queries (`protect.Equality`) | | `unique(downcase)` | Case-insensitive exact match | -| `match` | Full-text search (ngram tokenizer, k=6, m=2048) | -| `match(tokenizer=standard)` | Full-text with word-boundary tokenizer | -| `ore` | Range queries (`<`, `>`, `BETWEEN`, `ORDER BY`) | -| `ste_vec(prefix=t/c)` | JSON path and containment queries | +| `match` | Full-text search (`protect.FreeTextSearch`) — ngram tokenizer, token length 3, k=6, m=2048 by default | +| `match(k=8,m=1024,tokenizer=standard,token_length=3,include_original=true)` | Full-text search with tuned parameters | +| `ore` | Range queries and sorting (`protect.OrderAndRange`) | +| `ste_vec(prefix=table/column)` | JSON path and containment queries (`protect.JSONSelector`, `protect.JSONContains`) — forces the column to `json` | #### Type inference -Cast type is inferred from the Go field type. Override with `cast=`: +The storage type is inferred from the Go field type. Override with +`cast=`: -| Go type | Inferred cast | Override example | +| Go type | Inferred type | Common overrides | |---|---|---| -| `string` | `string` | `cast=text` | -| `int`, `float64` | `number` | `cast=bigint` | +| `string` | `text` | `cast=date`, `cast=timestamp`, `cast=json` | +| signed/unsigned ints | `big_int` | `cast=int`, `cast=small_int` | +| `float32`, `float64` | `float` | `cast=decimal` (exact, no float rounding) | | `bool` | `boolean` | | -| `map`, `any`, `[]T` | `json` | | +| `time.Time` | `timestamp` | `cast=date` | +| `map`, `any`, slices | `json` | | + +The full set of types is `text`, `big_int`, `int`, `small_int`, `float`, +`decimal`, `boolean`, `date`, `timestamp`, and `json` (constants +`protect.CastAsText` … `protect.CastAsJSON`). The legacy names `string` +(→ `text`) and `number` (→ `float`) are still accepted and normalized +automatically. ### Programmatic builder -For complex schemas or when you prefer type safety over struct tags: +For dynamic schemas or when you prefer builders over tags: ```go users := protect.NewSchema("users"). - Column("email", protect.CastAsString).Equality().FreeTextSearch().Done(). - Column("name", protect.CastAsString).FreeTextSearch().Done(). - Column("age", protect.CastAsNumber).OrderAndRange().Done(). - Column("profile", protect.CastAsJson).SearchableJSON("users/profile").Done(). + Column("email", protect.CastAsText).Equality(protect.TokenFilter{Kind: "downcase"}).FreeTextSearch().Done(). + Column("name", protect.CastAsText).FreeTextSearch(protect.WithK(8), protect.WithM(1024)).Done(). + Column("age", protect.CastAsBigInt).OrderAndRange().Done(). + Column("salary", protect.CastAsDecimal).Done(). + Column("profile", protect.CastAsJSON).SearchableJSON("users/profile").Done(). Build() ``` @@ -186,35 +290,66 @@ client, err := protect.NewClient(ctx, protect.WithCredentials(crn, accessKey, clientID, clientKey), ) -// Multi-tenant keyset isolation +// Multi-tenant keyset isolation — scope this client to one tenant's keys client, err := protect.NewClient(ctx, protect.WithSchemas(users), - protect.WithKeyset("tenant-a"), + protect.WithKeyset("tenant-a"), // by name, or WithKeysetID("") ) defer client.Close() ``` +`Client` is safe for concurrent use. `Close` releases the native resources; +operations on a closed client return `ErrClientClosed`. + +### Ciphertext format version + +The SDK can produce two on-disk payload formats. The default, +`EncryptedFormatV2`, matches databases provisioned with the v2 database +schema. Select `EncryptedFormatV3` for databases provisioned with the v3 +schema (typed encrypted columns — see [PostgreSQL setup](#postgresql-setup)): + +```go +client, err := protect.NewClient(ctx, + protect.WithSchemas(users), + protect.WithEncryptedFormat(protect.EncryptedFormatV3), +) +``` + +The two formats' search terms are not interchangeable — pick the format that +matches your database schema. Decryption accepts both formats regardless of +this setting, so you can read old data while writing new-format data during a +migration. + +> [!NOTE] +> V3 maps each column's index configuration onto a typed database column. A +> few combinations have no v3 equivalent (for example `unique` + `match` with +> no `ore` on a text column, or `boolean` columns with any index). Rather than +> silently dropping a search capability, `NewClient` fails with +> `ErrUnsupportedFormat` and a hint about what to change. + ## Encrypting and decrypting ### Models -The fastest way to encrypt data. Fields with a `cs` tag are encrypted; everything else passes through. +The fastest way to encrypt data. Fields with a `cs` tag are encrypted; +everything else passes through: ```go user := User{ID: 1, Email: "alice@example.com", Name: "Alice", Age: 28, Role: "admin"} -// Encrypt — returns map with *Encrypted values for tagged fields +// Encrypt — returns a map with encrypted values for tagged fields encrypted, err := client.EncryptModel(ctx, users, user) -// Decrypt — populates struct from encrypted map +// Decrypt — populates the struct from the encrypted map var decrypted User err = client.DecryptModel(ctx, users, encrypted, &decrypted) ``` ### Bulk models -Single KMS call for all fields across all models: +One keyservice round trip for all fields across all models — use this for +anything more than a single record: ```go encryptedModels, err := client.BulkEncryptModels(ctx, users, userSlice) @@ -240,86 +375,259 @@ items := []protect.PlaintextItem{ {Column: users.Column("email"), Plaintext: "bob@example.com"}, } encrypted, err := client.EncryptBulk(ctx, items) -``` -### With options +plaintexts, err := client.DecryptBulk(ctx, encrypted) -```go -lc := &protect.LockContext{IdentityClaim: []string{"user:123"}} +// Per-item error handling instead of all-or-nothing: +results, err := client.DecryptBulkFallible(ctx, encrypted) +for _, r := range results { + if r.Err != nil { /* this item failed */ } +} +``` -encrypted, err := client.Encrypt(ctx, users.Column("email"), "alice@example.com", - protect.WithLockContext(lc), -) +### Value types -plaintext, err := client.Decrypt(ctx, encrypted, - protect.WithLockContext(lc), -) -``` +What you can pass in, and what `Decrypt` hands back, by column type: -> [!CAUTION] -> Data encrypted with a lock context can only be decrypted with the same context. +| Column type | Accepted plaintext | `Decrypt` returns | +|---|---|---| +| `text` | `string` | `string` | +| `big_int`, `int`, `small_int` | any Go integer, or a whole `float64` | `json.Number` (exact, full `int64` range) | +| `float` | any Go number | `json.Number` | +| `decimal` | any Go number (stored exactly — `0.1` stays `0.1`) | `string` | +| `boolean` | `bool` | `bool` | +| `date` | `time.Time`, or `"YYYY-MM-DD"` / RFC 3339 `string` | `string` (`"YYYY-MM-DD"`) | +| `timestamp` | `time.Time`, or RFC 3339 `string` | `string` (RFC 3339) | +| `json` | `map[string]any`, slices, anything JSON-marshalable | `map[string]any` / `[]any` | + +Integer conversions are exact-or-error: fractional, out-of-range, `NaN`, and +`Inf` values are rejected rather than truncated. `DecryptModel` and +`BulkDecryptModels` convert these raw values back into your struct's field +types (including `time.Time` and all integer widths) automatically. ## Querying encrypted data -Encrypt search terms to query encrypted columns without exposing plaintext: +Encrypt search terms to query encrypted columns without exposing plaintext. +`EncryptQuery` returns an opaque `*protect.QueryTerm` — bind it directly as a +SQL parameter. Depending on the column configuration and format version, a +term may serialize as a JSON object or a bare JSON string; always treat it as +opaque. ```go // Exact match -query, _ := client.EncryptQuery(ctx, users.Column("email"), protect.Equality, "alice@example.com") -// Use query.UniqueIndex in SQL +term, err := client.EncryptQuery(ctx, users.Column("email"), protect.Equality, "alice@example.com") // Full-text search -query, _ := client.EncryptQuery(ctx, users.Column("name"), protect.FreeTextSearch, "alice") -// Use query.MatchIndex in SQL +term, err = client.EncryptQuery(ctx, users.Column("name"), protect.FreeTextSearch, "alice") + +// Range comparison (works for numbers, dates, timestamps) +term, err = client.EncryptQuery(ctx, users.Column("age"), protect.OrderAndRange, 25) -// Range comparison -query, _ := client.EncryptQuery(ctx, users.Column("age"), protect.OrderAndRange, 25) -// Use query.OreIndex in SQL +// JSON containment — does the document contain this structure? +term, err = client.EncryptQuery(ctx, users.Column("metadata"), protect.JSONContains, + map[string]any{"role": "admin"}) -// JSON containment -query, _ := client.EncryptQuery(ctx, users.Column("metadata"), protect.JSONContains, map[string]any{"role": "admin"}) +// JSON path — target one field of the document +term, err = client.EncryptQuery(ctx, users.Column("metadata"), protect.JSONSelector, "$.role") -// Bulk queries -queries, _ := client.EncryptQueryBulk(ctx, []protect.QueryItem{ +// Inspect the raw payload if needed +_ = term.String() // or term.Bytes() + +// Bulk — one keyservice round trip for many terms +terms, err := client.EncryptQueryBulk(ctx, []protect.QueryItem{ {Column: users.Column("email"), QueryType: protect.Equality, Plaintext: "alice@example.com"}, {Column: users.Column("name"), QueryType: protect.FreeTextSearch, Plaintext: "bob"}, }) ``` -| Query Type | Use Case | +| Query type | Requires directive | SQL shape | +|---|---|---| +| `protect.Equality` | `unique` | `WHERE col = $1` | +| `protect.FreeTextSearch` | `match` | `WHERE col LIKE $1` (v2) / `WHERE col @> $1` (v3) | +| `protect.OrderAndRange` | `ore` | `WHERE col > $1`, `ORDER BY` | +| `protect.JSONSelector` | `ste_vec` | `WHERE col -> $1 IS NOT NULL` | +| `protect.JSONContains` | `ste_vec` | `WHERE col @> $1` | + +See [PostgreSQL setup](#postgresql-setup) for the exact SQL, including the +casts each format version needs. + +## Identity-aware encryption + +The recommended way to bind data access to end users is +[OIDC federation](#per-user-identity-with-oidc-federation): authentication, +authorization, and audit attribution all follow the federated user with no +per-operation code. + +### Lock context + +A lock context goes further and ties **individual ciphertexts** to identity +claims — the same claims must be presented to decrypt: + +```go +lc := &protect.LockContext{IdentityClaim: []string{"sub"}} + +encrypted, err := client.Encrypt(ctx, users.Column("email"), "secret", + protect.WithLockContext(lc)) + +plaintext, err := client.Decrypt(ctx, encrypted, + protect.WithLockContext(lc)) +``` + +> [!IMPORTANT] +> Lock contexts require the client to authenticate with an identity-bearing +> token — that is, `WithOIDCFederation`. With plain access-key authentication +> the platform rejects lock-context operations as forbidden. + +> [!CAUTION] +> Data encrypted with a lock context can only be decrypted with the same +> context. Losing the identity claims means losing the data. + +### Audit context + +Attach arbitrary application context to key operations for your CipherStash +audit logs (informational — not verified, and not part of the key +derivation): + +```go +encrypted, err := client.Encrypt(ctx, users.Column("email"), "alice@example.com", + protect.WithAuditContext(map[string]any{"request_id": reqID, "actor": "billing-service"}), +) +``` + +## PostgreSQL setup + +Searchable encryption requires the CipherStash encrypted-column database +extension — plain SQL, no superuser or native extension install needed. The +names below (`eql_v2_encrypted`, `eql_v3_*`, and the `eql_v3.query_*` casts) +are defined by that extension; use them exactly as shown. + +### v2 schema (default, `EncryptedFormatV2`) + +```bash +curl -fsSLO https://github.com/cipherstash/encrypt-query-language/releases/download/eql-2.3.1/cipherstash-encrypt.sql +psql -f cipherstash-encrypt.sql +``` + +Every encrypted column uses the one generic column type: + +```sql +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + email eql_v2_encrypted, + age eql_v2_encrypted +); +``` + +Insert ciphertexts and bind query terms as `jsonb`: + +```sql +INSERT INTO users (email, age) VALUES ($1::jsonb, $2::jsonb); + +SELECT email::jsonb FROM users WHERE email = $1::jsonb; -- Equality +SELECT email::jsonb FROM users WHERE email LIKE $1::jsonb; -- FreeTextSearch +SELECT age::jsonb FROM users WHERE age > $1::jsonb + ORDER BY eql_v2.order_by(age) ASC; -- OrderAndRange +``` + +### v3 schema (`EncryptedFormatV3`) + +```bash +curl -fsSLO https://github.com/cipherstash/encrypt-query-language/releases/download/eql-3.0.0/cipherstash-encrypt.sql +psql -f cipherstash-encrypt.sql +``` + +v3 replaces the generic column type with typed columns, so the database +enforces exactly which search capabilities each column carries. Pick the +column type from your schema definition: + +| Go schema configuration | v3 column type | |---|---| -| `protect.Equality` | Exact match (`=`) | -| `protect.FreeTextSearch` | Substring/fuzzy search | -| `protect.OrderAndRange` | Range comparisons, sorting | -| `protect.JSONSelector` | JSON path queries (`$.field`) | -| `protect.JSONContains` | JSON containment (`@>`) | +| no indexes (storage only) | `eql_v3_` | +| `Equality` | `eql_v3__eq` | +| `OrderAndRange` (with or without `Equality`) | `eql_v3__ord_ore` | +| `FreeTextSearch` only (text) | `eql_v3_text_match` | +| `Equality` + `FreeTextSearch` + `OrderAndRange` (text) | `eql_v3_text_search_ore` | +| `SearchableJSON` | `eql_v3_json` | + +where `` is `text`, `bigint`, `integer`, `smallint`, `double`, +`numeric`, `boolean`, `date`, or `timestamp`, matching the column's cast +type (`big_int` → `bigint`, `int` → `integer`, `float` → `double`, +`decimal` → `numeric`). + +```sql +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + email eql_v3_text_eq, -- unique + age eql_v3_bigint_ord_ore, -- ore + bio eql_v3_text_search_ore, -- unique + match + ore + metadata eql_v3_json -- ste_vec +); +``` + +Insert ciphertexts as `jsonb`; cast query terms to the matching +`eql_v3.query_` type (the column type name without the +`eql_v3_` prefix): + +```sql +INSERT INTO users (email, age) VALUES ($1::jsonb, $2::jsonb); + +-- Equality +SELECT email::jsonb FROM users +WHERE email = $1::jsonb::eql_v3.query_text_eq; + +-- OrderAndRange +SELECT age::jsonb FROM users +WHERE age > $1::jsonb::eql_v3.query_bigint_ord_ore +ORDER BY eql_v3.ord_term_ore(age) ASC; + +-- FreeTextSearch +SELECT bio::jsonb FROM users +WHERE bio @> $1::jsonb::eql_v3.query_text_search_ore; + +-- JSONContains +SELECT metadata::jsonb FROM users +WHERE metadata @> $1::jsonb::eql_v3.query_jsonb; + +-- JSONSelector: the term is a bare string — bind it as text +SELECT metadata -> $1::text FROM users +WHERE metadata -> $1::text IS NOT NULL; +``` + +Every operator also has a callable function equivalent (`eql_v3.eq(...)`, +`eql_v3.lt(...)`, …) for systems that expose the database through RPC. ## Error handling All errors support `errors.Is()` for programmatic handling: ```go -_, err := client.Encrypt(ctx, users.Column("email"), "value") - -if errors.Is(err, protect.ErrUnknownColumn) { - // column not in schema -} -if errors.Is(err, protect.ErrMissingIndex) { - // index not configured for this query type -} -if errors.Is(err, protect.ErrClientClosed) { +_, err := client.EncryptQuery(ctx, users.Column("email"), protect.OrderAndRange, "x") + +switch { +case errors.Is(err, protect.ErrMissingIndex): + // the column has no `ore` directive +case errors.Is(err, protect.ErrUnknownColumn): + // column not in any registered schema +case errors.Is(err, protect.ErrClientClosed): // client was already closed } ``` -| Sentinel | Description | +| Sentinel | Meaning | |---|---| -| `ErrUnknownColumn` | Column not found in encryption schema | -| `ErrMissingIndex` | Required index not configured | -| `ErrInvalidQueryInput` | Wrong value type for query operation | -| `ErrInvalidJSONPath` | Invalid JSON path for selector query | +| `ErrUnknownColumn` | Column not found in the encryption schema | +| `ErrMissingIndex` | The query type needs an index directive the column doesn't have | +| `ErrInvalidQueryInput` | Wrong value type for the query operation | +| `ErrInvalidJSONPath` | Invalid JSON path for a selector query (paths start with `$`) | +| `ErrInvalidCiphertext` | Value is not a valid ciphertext | +| `ErrUnsupportedFormat` | The column's index configuration has no equivalent in the selected format version | +| `ErrAuthStrategy` | Authentication strategy misconfigured (e.g. OIDC federation without a workspace CRN, or combined with a token provider) | +| `ErrSteVecRequiresJSON` | A JSON-search directive on a non-`json` column | | `ErrClientClosed` | Client has been closed | +Errors are `*protect.Error` values carrying the failing operation +(`Encrypt`, `NewClient`, …) and the underlying cause via `Unwrap`. + ## API reference ### Schema @@ -327,8 +635,10 @@ if errors.Is(err, protect.ErrClientClosed) { ```go func TableSchema(tableName string, model any) (*TableDef, error) func NewSchema(tableName string) *SchemaBuilder -func (td *TableDef) Column(name string) ColumnRef + func (td *TableDef) Name() string +func (td *TableDef) Column(name string) ColumnRef // panics on unknown column +func (td *TableDef) ColumnOK(name string) (ColumnRef, bool) ``` ### Client @@ -341,60 +651,47 @@ func (c *Client) Close() error func WithSchemas(schemas ...*TableDef) ClientOption func WithCredentials(workspaceCRN, accessKey, clientID, clientKey string) ClientOption func WithKeyset(name string) ClientOption +func WithKeysetID(id string) ClientOption +func WithOIDCFederation(getToken func(ctx context.Context) (string, error)) ClientOption +func WithTokenProvider(getToken func(ctx context.Context) (string, error)) ClientOption +func WithEncryptedFormat(f EncryptedFormat) ClientOption // EncryptedFormatV2 (default) | EncryptedFormatV3 ``` ### Operations ```go -func (c *Client) Encrypt(ctx, col, plaintext, ...Option) (*Encrypted, error) -func (c *Client) Decrypt(ctx, encrypted, ...Option) (any, error) -func (c *Client) EncryptBulk(ctx, items, ...Option) ([]Encrypted, error) -func (c *Client) DecryptBulk(ctx, items, ...Option) ([]any, error) -func (c *Client) DecryptBulkFallible(ctx, items, ...Option) ([]DecryptResult, error) -func (c *Client) EncryptQuery(ctx, col, queryType, plaintext, ...Option) (*Encrypted, error) -func (c *Client) EncryptQueryBulk(ctx, queries, ...Option) ([]Encrypted, error) - -// Options +func (c *Client) Encrypt(ctx context.Context, col ColumnRef, plaintext any, opts ...Option) (*Encrypted, error) +func (c *Client) Decrypt(ctx context.Context, e *Encrypted, opts ...Option) (any, error) +func (c *Client) EncryptBulk(ctx context.Context, items []PlaintextItem, opts ...Option) ([]Encrypted, error) +func (c *Client) DecryptBulk(ctx context.Context, items []*Encrypted, opts ...Option) ([]any, error) +func (c *Client) DecryptBulkFallible(ctx context.Context, items []*Encrypted, opts ...Option) ([]DecryptResult, error) +func (c *Client) EncryptQuery(ctx context.Context, col ColumnRef, qt QueryType, plaintext any, opts ...Option) (*QueryTerm, error) +func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts ...Option) ([]*QueryTerm, error) + +// Per-operation options func WithLockContext(lc *LockContext) Option -func WithServiceToken(token string) Option func WithAuditContext(ctx any) Option ``` ### Models ```go -func (c *Client) EncryptModel(ctx, schema, model) (map[string]any, error) -func (c *Client) DecryptModel(ctx, schema, data, dest) error -func (c *Client) BulkEncryptModels(ctx, schema, models) ([]map[string]any, error) -func (c *Client) BulkDecryptModels(ctx, schema, data, dest) error +func (c *Client) EncryptModel(ctx context.Context, schema *TableDef, model any) (map[string]any, error) +func (c *Client) DecryptModel(ctx context.Context, schema *TableDef, data map[string]any, dest any) error +func (c *Client) BulkEncryptModels(ctx context.Context, schema *TableDef, models any) ([]map[string]any, error) +func (c *Client) BulkDecryptModels(ctx context.Context, schema *TableDef, data []map[string]any, dest any) error ``` ### Utilities ```go -func IsEncrypted(value any) bool -``` - -## PostgreSQL setup - -Searchable encryption requires the EQL extension: - -```bash -curl -sLo cipherstash-encrypt.sql \ - https://github.com/cipherstash/encrypt-query-language/releases/latest/download/cipherstash-encrypt.sql -psql -f cipherstash-encrypt.sql -``` - -```sql -CREATE TABLE users ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v2_encrypted -); +func IsEncrypted(value any) bool // true for stored ciphertexts (either format); false for query terms ``` ## Prebuilt libraries -The SDK ships with precompiled static libraries for all supported platforms. No Rust toolchain required. +The SDK ships with precompiled static libraries for all supported platforms. +No Rust toolchain required. | Platform | Library | |---|---| @@ -405,15 +702,25 @@ The SDK ships with precompiled static libraries for all supported platforms. No | Linux ARM64 (musl) | `libprotect_ffi_linux_arm64_musl.a` | | Linux x64 (musl) | `libprotect_ffi_linux_x64_musl.a` | +On Alpine Linux (musl libc), build with the `musl` tag: + +```bash +go build -tags=musl ./... +``` + ## Running the examples ```bash -go run examples/basic_usage.go +cp .env.example .env # fill in your workspace credentials +go run ./examples # Alpine Linux / musl -go run -tags=musl examples/basic_usage.go +go run -tags=musl ./examples ``` +See [DEVELOPMENT.md](DEVELOPMENT.md) for building the native library from +source. + --- [Missing something?](https://github.com/cipherstash/protectgo/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20README.md) diff --git a/crates/protect-ffi-c/Cargo.toml b/crates/protect-ffi-c/Cargo.toml index 83b5d20..051128a 100644 --- a/crates/protect-ffi-c/Cargo.toml +++ b/crates/protect-ffi-c/Cargo.toml @@ -8,11 +8,11 @@ name = "protect_ffi" crate-type = ["cdylib", "staticlib"] [dependencies] -cipherstash-client = { version = "=0.34.1-alpha.2", features = ["tokio"] } -cipherstash-config = { version = "=0.34.1-alpha.2" } -cipherstash-core = { version = "=0.34.1-alpha.2" } -cts-common = { version = "=0.34.1-alpha.2", default-features = false } -stack-profile = { version = "=0.34.1-alpha.2" } +cipherstash-client = { version = "=0.40.0", features = ["tokio"] } +cts-common = { version = "=0.40.0", default-features = false } +stack-profile = { version = "=0.40.0" } +stack-auth = { version = "=0.40.0" } +eql-bindings = { version = "=3.0.0" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } @@ -20,6 +20,8 @@ thiserror = "2.0" hex = "0.4" once_cell = "1.20" rust_decimal = "1.37" +chrono = { version = "0.4", default-features = false, features = ["serde", "alloc"] } +libc = "0.2" [build-dependencies] cbindgen = "0.26" diff --git a/crates/protect-ffi-c/src/auth.rs b/crates/protect-ffi-c/src/auth.rs new file mode 100644 index 0000000..60b27b3 --- /dev/null +++ b/crates/protect-ffi-c/src/auth.rs @@ -0,0 +1,390 @@ +//! Callback-driven authentication strategies for the C FFI boundary. +//! +//! protect-ffi's Go caller can supply a `getToken` callback (a C function +//! pointer plus an opaque `cgo.Handle`) that produces either a CTS service +//! token directly (`tokenProvider`) or a third-party OIDC JWT to federate +//! (`oidcFederation`). This module wraps that callback in the +//! [`stack_auth::AuthStrategy`] / [`stack_auth::OidcProvider`] traits so the +//! rest of the client is agnostic to how tokens are sourced. +//! +//! The callback may perform network I/O in Go, so it is always invoked from +//! [`tokio::task::spawn_blocking`]. It returns a C-heap (malloc'd) NUL-terminated +//! JSON string that Rust copies and frees with [`libc::free`]; a NULL return +//! means "the provider failed with no detail". + +use std::ffi::CStr; +use std::future::Future; +use std::os::raw::c_char; + +use serde::Deserialize; +use serde_json::Value; +use stack_auth::{ + AuthError, AuthStrategy, AutoStrategy, OidcFederationStrategy, OidcProvider, SecretToken, + ServerError, ServiceToken, +}; + +/// The C `getToken` callback: `char *(*)(uint64_t handle)`. +/// +/// Go returns a malloc'd (C heap, via `C.CString`) NUL-terminated JSON string, +/// or NULL to signal "provider failed with no detail". +pub type ProtectTokenFn = Option *mut c_char>; + +/// A resolved token callback: the raw C function pointer and the opaque handle +/// passed back to it verbatim on every invocation. +/// +/// A bare function pointer plus a `u64` is `Send + Sync + Copy`, so this can be +/// held inside a long-lived client and moved into blocking tasks freely. +#[derive(Clone, Copy)] +pub(crate) struct GoTokenCallback { + get_token: unsafe extern "C" fn(u64) -> *mut c_char, + handle: u64, +} + +impl GoTokenCallback { + pub(crate) fn new(get_token: unsafe extern "C" fn(u64) -> *mut c_char, handle: u64) -> Self { + Self { get_token, handle } + } +} + +/// The auth strategy declared in `newClient` options. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct AuthStrategyOpts { + #[serde(rename = "type")] + pub strategy_type: AuthStrategyType, + /// Optional CTS base URL override (oidcFederation only). + pub base_url: Option, +} + +#[derive(Deserialize, PartialEq, Eq, Clone, Copy)] +pub(crate) enum AuthStrategyType { + #[serde(rename = "oidcFederation")] + OidcFederation, + #[serde(rename = "tokenProvider")] + TokenProvider, +} + +/// Invoke the Go callback on a blocking thread and return its JSON result. +/// +/// `None` means the callback returned NULL (or the blocking task panicked). +async fn invoke_token_callback(cb: GoTokenCallback) -> Option { + tokio::task::spawn_blocking(move || { + // SAFETY: `cb.get_token` is a valid C function pointer supplied by the + // Go caller; `cb.handle` is the opaque value it expects back. + let ptr = unsafe { (cb.get_token)(cb.handle) }; + if ptr.is_null() { + return None; + } + // SAFETY: Go guarantees a NUL-terminated C string when the pointer is + // non-null. + let owned = unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned(); + // SAFETY: the string was allocated by Go's C allocator (`C.CString` → + // `malloc`), so it must be released with `free`. + unsafe { libc::free(ptr as *mut libc::c_void) }; + Some(owned) + }) + .await + .unwrap_or(None) +} + +/// A protect-ffi-internal error for a callback that returned a malformed shape +/// (not an auth-domain outcome). Kept as `Server` to match the shape such +/// protocol violations took before typed reconstruction existed. +fn strategy_protocol_error(msg: impl Into) -> AuthError { + AuthError::Server(ServerError(msg.into())) +} + +/// Build an attributable message for a reconstructed auth failure. Falls back +/// to the failure `code` (or a generic phrase when that is absent too) so a +/// reconstructed error is never blank. Codes that map to a fixed [`AuthError`] +/// variant ignore this message; it only surfaces for the `Custom` fallthrough. +fn auth_failure_message(code: &str, message: String) -> String { + if !message.is_empty() { + message + } else if !code.is_empty() { + format!("auth failure: {code}") + } else { + "strategy.getToken failed with an unspecified auth failure".to_string() + } +} + +/// Reconstruct a [`stack_auth::AuthError`] from a `{ ...payload, type, error, +/// help?, url? }` failure object via [`AuthError::from_error_code`], so a +/// strategy failure crosses back into Rust as the real typed error — +/// preserving its code and any structured payload — rather than a flattened +/// `Server`. Unknown / foreign codes fall through to `AuthError::Custom`. +fn failure_to_auth_error(failure: &Value) -> AuthError { + let obj = match failure.as_object() { + Some(o) => o, + None => return strategy_protocol_error("strategy.getToken failed"), + }; + let code = obj.get("type").and_then(Value::as_str).unwrap_or_default(); + // The message lives on the nested `error` object. + let message = obj + .get("error") + .and_then(Value::as_object) + .and_then(|err| err.get("message")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + // Thread the structured payload: the whole failure object minus the + // reserved keys. + let mut payload = obj.clone(); + for key in ["type", "error", "help", "url"] { + let _ = payload.remove(key); + } + AuthError::from_error_code(code, auth_failure_message(code, message), &payload) +} + +/// Decode the callback's JSON envelope into a bare token string, or a typed +/// [`AuthError`]. +/// +/// Accepts the success form `{"token": ""}` and the failure form +/// `{"failure": {...}}`. Every protocol violation maps to a precise +/// `AuthError::Server(ServerError(..))` message the contract pins. +fn decode_token_envelope(raw: Option) -> Result { + let raw = raw.ok_or_else(|| strategy_protocol_error("strategy callback returned no result"))?; + let value: Value = serde_json::from_str(&raw) + .map_err(|_| strategy_protocol_error("strategy callback did not return an object"))?; + let obj = value + .as_object() + .ok_or_else(|| strategy_protocol_error("strategy callback did not return an object"))?; + + if let Some(failure) = obj.get("failure") { + return Err(failure_to_auth_error(failure)); + } + + let token = obj + .get("token") + .ok_or_else(|| strategy_protocol_error("strategy callback result missing 'token' field"))?; + let token = token + .as_str() + .ok_or_else(|| strategy_protocol_error("strategy callback 'token' field is not a string"))?; + Ok(token.to_string()) +} + +/// [`OidcProvider`] that fetches a third-party OIDC JWT from the Go callback. +/// +/// The callback's `token` is the raw third-party JWT (Clerk/Auth0/etc.); +/// [`OidcFederationStrategy`] exchanges it for a CTS service token. +pub(crate) struct GoOidcProvider { + cb: GoTokenCallback, +} + +impl GoOidcProvider { + pub(crate) fn new(cb: GoTokenCallback) -> Self { + Self { cb } + } +} + +impl OidcProvider for GoOidcProvider { + fn fetch(&self) -> impl Future> + Send { + let cb = self.cb; + async move { + let raw = invoke_token_callback(cb).await; + let token = decode_token_envelope(raw)?; + Ok(SecretToken::new(token)) + } + } +} + +/// [`AuthStrategy`] that treats the callback's `token` as a CTS service token +/// used directly. Caching is the Go side's responsibility — the callback is +/// invoked per ZeroKMS request. +pub(crate) struct GoProvidedTokenStrategy { + cb: GoTokenCallback, +} + +impl GoProvidedTokenStrategy { + pub(crate) fn new(cb: GoTokenCallback) -> Self { + Self { cb } + } +} + +impl AuthStrategy for &GoProvidedTokenStrategy { + async fn get_token(self) -> Result { + let raw = invoke_token_callback(self.cb).await; + let token = decode_token_envelope(raw)?; + Ok(ServiceToken::new(SecretToken::new(token))) + } +} + +/// The auth strategy held by the client: the filesystem/env-backed +/// [`AutoStrategy`], the OIDC federation strategy, or the direct +/// callback-supplied token strategy. +/// +/// `AutoStrategy` is boxed because it is substantially larger than the other +/// variants (clippy's `large_enum_variant`). +pub(crate) enum GoAuthStrategy { + Auto(Box), + Oidc(Box>), + Provided(GoProvidedTokenStrategy), +} + +impl AuthStrategy for &GoAuthStrategy { + async fn get_token(self) -> Result { + match self { + GoAuthStrategy::Auto(s) => (&**s).get_token().await, + GoAuthStrategy::Oidc(s) => (&**s).get_token().await, + GoAuthStrategy::Provided(s) => s.get_token().await, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + // A set of C callbacks returning fixed JSON envelopes, used to exercise the + // decode protocol end-to-end through the same free/copy path production uses. + unsafe extern "C" fn cb_bare_token(_h: u64) -> *mut c_char { + CString::new(r#"{"token":"the-service-token"}"#) + .unwrap() + .into_raw() + } + unsafe extern "C" fn cb_null(_h: u64) -> *mut c_char { + std::ptr::null_mut() + } + unsafe extern "C" fn cb_not_object(_h: u64) -> *mut c_char { + CString::new("42").unwrap().into_raw() + } + unsafe extern "C" fn cb_missing_token(_h: u64) -> *mut c_char { + CString::new(r#"{"nope":1}"#).unwrap().into_raw() + } + unsafe extern "C" fn cb_non_string_token(_h: u64) -> *mut c_char { + CString::new(r#"{"token":123}"#).unwrap().into_raw() + } + unsafe extern "C" fn cb_malformed(_h: u64) -> *mut c_char { + CString::new("{ not json").unwrap().into_raw() + } + unsafe extern "C" fn cb_failure_known(_h: u64) -> *mut c_char { + CString::new(r#"{"failure":{"type":"ACCESS_DENIED","error":{"message":"nope"}}}"#) + .unwrap() + .into_raw() + } + unsafe extern "C" fn cb_failure_unknown(_h: u64) -> *mut c_char { + CString::new(r#"{"failure":{"type":"WEIRD_CODE","error":{"message":"boom"}}}"#) + .unwrap() + .into_raw() + } + unsafe extern "C" fn cb_failure_no_message(_h: u64) -> *mut c_char { + CString::new(r#"{"failure":{"type":"WEIRD_CODE"}}"#) + .unwrap() + .into_raw() + } + + // These callbacks use CString::into_raw (Rust allocator), but the decode + // path frees with libc::free. To exercise the pure decode logic without a + // cross-allocator free, the tests below call decode_token_envelope directly + // with the JSON the callback would return. + + fn envelope(json: &str) -> Result { + decode_token_envelope(Some(json.to_string())) + } + + #[test] + fn bare_token_decodes() { + assert_eq!( + envelope(r#"{"token":"abc"}"#).unwrap(), + "abc".to_string() + ); + } + + // AuthError::Server prepends "Server error: " in its Display; Go matches on + // the precise inner substring, so assert containment (and the Server code). + + #[test] + fn null_result_is_a_precise_server_error() { + let err = decode_token_envelope(None).unwrap_err(); + assert_eq!(err.error_code(), "SERVER_ERROR"); + assert!(err.to_string().contains("strategy callback returned no result")); + } + + #[test] + fn non_object_is_a_precise_server_error() { + let err = envelope("42").unwrap_err(); + assert!(err + .to_string() + .contains("strategy callback did not return an object")); + } + + #[test] + fn malformed_json_is_a_precise_server_error() { + let err = envelope("{ not json").unwrap_err(); + assert!(err + .to_string() + .contains("strategy callback did not return an object")); + } + + #[test] + fn missing_token_is_a_precise_server_error() { + let err = envelope(r#"{"nope":1}"#).unwrap_err(); + assert!(err + .to_string() + .contains("strategy callback result missing 'token' field")); + } + + #[test] + fn non_string_token_is_a_precise_server_error() { + let err = envelope(r#"{"token":123}"#).unwrap_err(); + assert!(err + .to_string() + .contains("strategy callback 'token' field is not a string")); + } + + #[test] + fn known_failure_code_maps_to_typed_variant() { + let err = envelope(r#"{"failure":{"type":"ACCESS_DENIED","error":{"message":"nope"}}}"#) + .unwrap_err(); + assert_eq!(err.error_code(), "ACCESS_DENIED"); + assert!(!matches!(err, AuthError::Custom(_))); + } + + #[test] + fn unknown_failure_code_falls_back_to_custom_with_message() { + let err = envelope(r#"{"failure":{"type":"WEIRD_CODE","error":{"message":"boom"}}}"#) + .unwrap_err(); + assert_eq!(err.error_code(), "CUSTOM"); + assert_eq!(err.to_string(), "boom"); + } + + #[test] + fn failure_without_message_uses_code_in_message() { + let err = envelope(r#"{"failure":{"type":"WEIRD_CODE"}}"#).unwrap_err(); + assert_eq!(err.error_code(), "CUSTOM"); + assert_eq!(err.to_string(), "auth failure: WEIRD_CODE"); + } + + // Exercise the full C-callback path (fn-ptr invocation + copy) for the + // callbacks that use a Rust-allocated string. We leak rather than free to + // avoid a cross-allocator free in the test (production frees Go's malloc'd + // string with libc::free). + #[tokio::test] + async fn invoke_reads_bare_token_from_c_callback() { + let cb = GoTokenCallback::new(cb_bare_token, 0); + // Manually reproduce invoke without the libc::free (Rust-allocated). + let ptr = unsafe { (cb.get_token)(cb.handle) }; + let s = unsafe { CString::from_raw(ptr) }.to_string_lossy().into_owned(); + assert_eq!( + decode_token_envelope(Some(s)).unwrap(), + "the-service-token" + ); + } + + // Reference the remaining callbacks so they are not dead code; each is a + // valid extern "C" fn pointer of the ProtectTokenFn shape. + #[test] + fn callbacks_are_valid_fn_pointers() { + let _: [unsafe extern "C" fn(u64) -> *mut c_char; 8] = [ + cb_null, + cb_not_object, + cb_missing_token, + cb_non_string_token, + cb_malformed, + cb_failure_known, + cb_failure_unknown, + cb_failure_no_message, + ]; + } +} diff --git a/crates/protect-ffi-c/src/encrypt_config.rs b/crates/protect-ffi-c/src/encrypt_config.rs deleted file mode 100644 index 6c90523..0000000 --- a/crates/protect-ffi-c/src/encrypt_config.rs +++ /dev/null @@ -1,597 +0,0 @@ -use super::Error; -use cipherstash_client::schema::{ - column::{ArrayIndexMode, Index, IndexType, TokenFilter, Tokenizer}, - ColumnConfig, ColumnType, -}; -use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, str::FromStr}; - -#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] -pub struct Identifier { - #[serde(rename = "t")] - pub table: String, - #[serde(rename = "c")] - pub column: String, -} - -impl Identifier { - pub fn new(table: S, column: S) -> Self - where - S: Into, - { - let table = table.into(); - let column = column.into(); - - Self { table, column } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Default)] -pub struct Tables(HashMap); - -impl IntoIterator for Tables { - type Item = (String, Table); - type IntoIter = std::collections::hash_map::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Default)] -pub struct Table(HashMap); - -impl IntoIterator for Table { - type Item = (String, Column); - type IntoIter = std::collections::hash_map::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Default)] -pub struct EncryptConfig { - #[serde(rename = "v")] - pub version: u32, - pub tables: Tables, -} - -#[derive(Debug, Default, Deserialize, Serialize, Clone, PartialEq)] -pub struct Column { - #[serde(default)] - cast_as: CastAs, - #[serde(default)] - indexes: Indexes, -} - -#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "lowercase")] -pub enum CastAs { - BigInt, - Boolean, - Date, - Number, - #[default] - String, - Text, - Json, -} - -#[derive(Debug, Deserialize, Serialize, Clone, Default, PartialEq)] -pub struct Indexes { - #[serde(rename = "ore")] - ore_index: Option, - #[serde(rename = "unique")] - unique_index: Option, - #[serde(rename = "match")] - match_index: Option, - #[serde(rename = "ste_vec")] - ste_vec_index: Option, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -pub struct OreIndexOpts {} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -pub struct MatchIndexOpts { - #[serde(default = "default_tokenizer")] - tokenizer: Tokenizer, - #[serde(default)] - token_filters: Vec, - #[serde(default = "default_k")] - k: usize, - #[serde(default = "default_m")] - m: usize, - #[serde(default)] - include_original: bool, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -pub struct SteVecIndexOpts { - prefix: String, - #[serde(default)] - term_filters: Vec, - #[serde(default)] - array_index_mode: ArrayIndexMode, -} - -fn default_tokenizer() -> Tokenizer { - Tokenizer::Standard -} - -fn default_k() -> usize { - 6 -} - -fn default_m() -> usize { - 2048 -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -pub struct UniqueIndexOpts { - #[serde(default)] - token_filters: Vec, -} - -impl From for ColumnType { - fn from(value: CastAs) -> Self { - match value { - CastAs::BigInt => ColumnType::BigInt, - CastAs::Boolean => ColumnType::Boolean, - CastAs::Date => ColumnType::Date, - CastAs::Number => ColumnType::Float, - CastAs::String => ColumnType::Utf8Str, - CastAs::Text => ColumnType::Utf8Str, - CastAs::Json => ColumnType::JsonB, - } - } -} - -impl FromStr for EncryptConfig { - type Err = Error; - - fn from_str(data: &str) -> Result { - let config = serde_json::from_str(data).map_err(Error::Parse)?; - Ok(config) - } -} - -impl EncryptConfig { - pub fn into_config_map(self) -> Result, super::Error> { - let mut map = HashMap::new(); - for (table_name, columns) in self.tables.into_iter() { - for (column_name, column) in columns.into_iter() { - let column_config = column.into_column_config(&table_name, &column_name)?; - let key = Identifier::new(&table_name, &column_name); - map.insert(key, column_config); - } - } - Ok(map) - } -} - -impl Column { - pub fn into_column_config( - self, - table_name: &str, - column_name: &str, - ) -> Result { - // Validate ste_vec requires cast_as: json - if self.indexes.ste_vec_index.is_some() && self.cast_as != CastAs::Json { - return Err(super::Error::SteVecRequiresJsonCastAs { - table: table_name.to_string(), - column: column_name.to_string(), - found_cast_as: cast_as_name(&self.cast_as).to_string(), - }); - } - - let mut config = ColumnConfig::build(column_name.to_string()).casts_as(self.cast_as.into()); - - if self.indexes.ore_index.is_some() { - config = config.add_index(Index::new_ore()); - } - - if let Some(opts) = self.indexes.match_index { - config = config.add_index(Index::new(IndexType::Match { - tokenizer: opts.tokenizer, - token_filters: opts.token_filters, - k: opts.k, - m: opts.m, - include_original: opts.include_original, - })); - } - - if let Some(opts) = self.indexes.unique_index { - config = config.add_index(Index::new(IndexType::Unique { - token_filters: opts.token_filters, - })) - } - - if let Some(SteVecIndexOpts { - prefix, - term_filters, - array_index_mode, - }) = self.indexes.ste_vec_index - { - config = config.add_index(Index::new(IndexType::SteVec { - prefix, - term_filters, - array_index_mode, - })) - } - - Ok(config) - } -} - -/// Get a human-readable name for CastAs value -fn cast_as_name(cast_as: &CastAs) -> &'static str { - match cast_as { - CastAs::BigInt => "bigint", - CastAs::Boolean => "boolean", - CastAs::Date => "date", - CastAs::Number => "number", - CastAs::String => "string", - CastAs::Text => "text", - CastAs::Json => "json", - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - fn parse(json: serde_json::Value) -> HashMap { - serde_json::from_value::(json) - .unwrap() - .into_config_map() - .unwrap() - } - - #[test] - fn column_with_empty_options_gets_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": {} - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!(column.cast_type, ColumnType::Utf8Str); - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_column_with_cast_as() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "favourite_int": { - "cast_as": "number" - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "favourite_int"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!(column.cast_type, ColumnType::Float); - assert_eq!(column.name, "favourite_int"); - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_empty_indexes() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": {} - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_ore_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "ore": {} - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!(column.indexes[0].index_type, IndexType::Ore); - } - - #[test] - fn can_parse_unique_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "unique": {} - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![] - } - ); - } - - #[test] - fn can_parse_unique_index_with_token_filter() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "unique": { - "token_filters": [ - { - "kind": "downcase" - } - ] - } - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![TokenFilter::Downcase] - } - ); - } - - #[test] - fn can_parse_match_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "match": {} - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Standard, - token_filters: vec![], - k: 6, - m: 2048, - include_original: false - } - ); - } - - #[test] - fn can_parse_match_index_with_all_opts_set() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "match": { - "tokenizer": { - "kind": "ngram", - "token_length": 3, - }, - "token_filters": [ - { - "kind": "downcase" - } - ], - "k": 8, - "m": 1024, - "include_original": true - } - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "email"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Ngram { token_length: 3 }, - token_filters: vec![TokenFilter::Downcase], - k: 8, - m: 1024, - include_original: true - } - ); - } - - #[test] - fn can_parse_ste_vec_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "event_data": { - "cast_as": "json", - "indexes": { - "ste_vec": { - "prefix": "event-data" - } - } - } - } - } - }); - - let encrypt_config = parse(json); - - let ident = Identifier::new("users", "event_data"); - - let column = encrypt_config.get(&ident).expect("column exists"); - - assert_eq!(column.cast_type, ColumnType::JsonB); - assert_eq!( - column.indexes[0].index_type, - IndexType::SteVec { - prefix: "event-data".into(), - term_filters: vec![], - array_index_mode: Default::default(), - }, - ); - } - - #[test] - fn ste_vec_with_non_json_cast_as_fails_validation() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "event_data": { - "cast_as": "string", - "indexes": { - "ste_vec": { - "prefix": "event-data" - } - } - } - } - } - }); - - let result = serde_json::from_value::(json) - .unwrap() - .into_config_map(); - - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("users"), - "Error should mention table name: {}", - err_msg - ); - assert!( - err_msg.contains("event_data"), - "Error should mention column name: {}", - err_msg - ); - assert!( - err_msg.contains("ste_vec"), - "Error should mention ste_vec index: {}", - err_msg - ); - assert!( - err_msg.contains("json"), - "Error should mention json cast_as requirement: {}", - err_msg - ); - } - - #[test] - fn ste_vec_with_json_cast_as_succeeds() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "event_data": { - "cast_as": "json", - "indexes": { - "ste_vec": { - "prefix": "event-data" - } - } - } - } - } - }); - - let result = serde_json::from_value::(json) - .unwrap() - .into_config_map(); - - assert!(result.is_ok(), "ste_vec with json cast_as should succeed"); - let config = result.unwrap(); - let ident = Identifier::new("users", "event_data"); - let column = config.get(&ident).expect("column exists"); - assert_eq!(column.cast_type, ColumnType::JsonB); - } -} diff --git a/crates/protect-ffi-c/src/eql_v3.rs b/crates/protect-ffi-c/src/eql_v3.rs new file mode 100644 index 0000000..d8f9e2a --- /dev/null +++ b/crates/protect-ffi-c/src/eql_v3.rs @@ -0,0 +1,1907 @@ +//! EQL v3 dual-format support. +//! +//! protect-ffi historically speaks the EQL v2.3 wire format (`{v: 2, k, i, c, +//! …}`). The `eql_v3` schema generation replaces the single +//! `eql_v2_encrypted` column type with per-capability column domains +//! (`public.eql_v3_text_eq`, `public.eql_v3_integer_ord_ore`, +//! `public.eql_v3_json`, …), their term-only query twins +//! (`eql_v3.query_text_eq`, `eql_v3.query_jsonb`, …) — unprefixed, since the +//! `eql_v3` schema already versions them — +//! and a new envelope: scalars are `{v: 3, i, c, }` with no `k` +//! discriminator; SteVec (encrypted JSONB) documents keep it +//! (`{v: 3, k: "sv", i, sv}`). +//! +//! Payloads are converted, not re-encrypted: cipherstash-client still emits +//! v2, and [`eql_bindings::from_v2`] rewrites the wire shape for the target +//! domain selected from the column configuration. Decryption accepts BOTH +//! formats regardless of the client's `eqlVersion` so data can be migrated +//! incrementally. + +use cipherstash_client::eql::{EqlCiphertext, EqlOutput, EqlQueryPayload, SteVecQueryTerm}; +use cipherstash_client::schema::{ + column::ColumnType, column::IndexType, column::SteVecMode, ColumnConfig, +}; +use cipherstash_client::zerokms::{self, EncryptedRecord, WithContext}; +use eql_bindings::from_v2::{from_v2_query_typed, from_v2_typed, is_v3_payload, TargetDomain}; +use eql_bindings::v3::domain_type::PUBLIC_TYPNAME_PREFIX; +use eql_bindings::v3::jsonb::SteVecDocument; +use eql_bindings::v3::terms::Selector; +use eql_bindings::v3::{DomainPayload, QueryPayload}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; + +use crate::Error; + +/// An EQL wire version this crate can emit. +/// +/// The version arrives as a raw `u8` from JavaScript (`newClient({ +/// eqlVersion })`) and is converted exactly once, at the FFI boundary, by +/// [`validate_eql_version`]. Everything downstream carries this enum, so +/// invalid versions are unrepresentable past that point and every +/// version-dependent branch is an exhaustive `match` the compiler checks +/// when a variant is added. +/// +/// The discriminants are the on-the-wire `v` values. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum EqlVersion { + V2 = 2, + V3 = 3, +} + +impl EqlVersion { + /// The wire version emitted when `eqlVersion` is omitted — v2, for + /// backwards compatibility. + pub(crate) const DEFAULT: Self = Self::V2; +} + +impl TryFrom for EqlVersion { + type Error = Error; + + fn try_from(version: u8) -> Result { + match version { + v if v == Self::V2 as u8 => Ok(Self::V2), + v if v == Self::V3 as u8 => Ok(Self::V3), + other => Err(Error::InvalidEqlVersion(other)), + } + } +} + +/// Validate the client-supplied `eqlVersion` option at the JS boundary: +/// only `2` and `3` are EQL wire versions this crate can emit. `None` +/// defaults to v2 for backwards compatibility. +pub(crate) fn validate_eql_version(version: Option) -> Result { + version.map_or(Ok(EqlVersion::DEFAULT), EqlVersion::try_from) +} + +/// The v2 index terms a column's configuration will produce on a stored +/// payload: `hm` from `unique`, `ob` from `ore`, `op` from `ope`, `bf` from +/// `match`, and the `sv` entry vector from `ste_vec`. +#[derive(Debug, Clone, Copy, Default)] +struct ConfiguredTerms { + hm: bool, + ob: bool, + op: bool, + bf: bool, + sv: bool, + /// The orderable-term primitive the `ste_vec` index emits, when `sv`. + /// `Compat` (the config default) emits CLLW-OPE `op`; the legacy + /// `Standard` emits CLLW-ORE `oc`. Only `op` is convertible to v3 — see + /// [`target_domain_for_column`]. + sv_mode: Option, +} + +impl ConfiguredTerms { + fn from_indexes(column_config: &ColumnConfig) -> Self { + let mut terms = Self::default(); + for index in &column_config.indexes { + match index.index_type { + IndexType::Unique { .. } => terms.hm = true, + IndexType::Ore => terms.ob = true, + IndexType::Ope => terms.op = true, + IndexType::Match { .. } => terms.bf = true, + IndexType::SteVec { mode, .. } => { + terms.sv = true; + terms.sv_mode = Some(mode); + } + } + } + terms + } + + fn any(&self) -> bool { + self.hm || self.ob || self.op || self.bf || self.sv + } +} + +/// The v3 domain family for a `cast_as` type. `None` for [`ColumnType::BigUInt`], +/// which no `cast_as` value maps to (fail closed if it ever appears). +fn v3_family(cast_type: ColumnType) -> Option<&'static str> { + match cast_type { + ColumnType::Text => Some("text"), + ColumnType::SmallInt => Some("smallint"), + ColumnType::Int => Some("integer"), + ColumnType::BigInt => Some("bigint"), + ColumnType::Float => Some("double"), + ColumnType::Decimal => Some("numeric"), + ColumnType::Date => Some("date"), + ColumnType::Timestamp => Some("timestamp"), + ColumnType::Boolean => Some("boolean"), + ColumnType::Json => Some("json"), + ColumnType::BigUInt => None, + } +} + +/// Qualify a bare family/suffix name with the version prefix every +/// public-schema column domain's typname carries (`text_eq` → +/// `eql_v3_text_eq`). Query twins stay unprefixed — eql-bindings strips it +/// back off when it derives `query_` from the stored domain. +fn v3_domain(bare: &str) -> String { + format!("{PUBLIC_TYPNAME_PREFIX}{bare}") +} + +fn no_v3_domain(column: &str, reason: impl Into, hint: impl Into) -> Error { + Error::NoV3Domain { + column: column.to_string(), + reason: reason.into(), + hint: hint.into(), + } +} + +/// Select the `eql_v3` column domain for a column, prefixed (`eql_v3_text_eq`). +/// +/// Every v2 index term is optional on the wire, so eql-bindings requires the +/// caller to name the target domain — this derives it from the column +/// configuration. Candidates are tried richest-first (`search_ore` > `search` > +/// `ord_ore` > `ord_ope` > `match` > `eq` > storage-only), and the winner must then cover +/// every configured CAPABILITY or the column errors ([`Error::NoV3Domain`]) +/// rather than silently stripping a term from stored rows: +/// +/// - equality (`unique`/`hm`) is covered by a domain carrying `hm`, `ob`, or +/// `op` — the ORE/OPE operators include `=`/`<>`, which is why non-text +/// `unique` + `ore` may select `_ord_ore` and drop `hm` without +/// losing anything; +/// - ordering (`ore`/`ob`, `ope`/`op`) is covered by `ob` or `op`, so +/// `unique` + `ore` + `ope` selects `_ord_ore` (ordering survives via +/// `ob`); +/// - match (`bf`) is covered only by a domain carrying `bf`, so text +/// combinations no single domain spans (`unique` + `match`, `ore` + `match`, +/// …) error instead of dropping a term; `unique` + `ore` + `match` reaches +/// `text_search_ore` and `unique` + `ope` + `match` reaches `text_search`; +/// - text ordering domains require `hm` alongside `ob`/`op`, so ordered text +/// without a `unique` index cannot be represented and errors; +/// - a column whose configured terms would ALL be dropped (bool with any +/// index, ordered-only text) errors rather than silently degrading to +/// storage-only. +pub(crate) fn target_domain_for_column(column_config: &ColumnConfig) -> Result { + let column = column_config.name.as_str(); + let terms = ConfiguredTerms::from_indexes(column_config); + let family = v3_family(column_config.cast_type).ok_or_else(|| { + no_v3_domain( + column, + format!( + "cast type {} has no EQL v3 domain family", + column_config.cast_type + ), + "Use eqlVersion 2 for this column.", + ) + })?; + + if family == "json" { + // eql_v3_json carries only sv. Upstream config accepts unique/ore/ope + // alongside ste_vec on a JSON column, so selecting the domain anyway + // would silently drop those terms — fail closed instead. + if terms.hm || terms.ob || terms.op || terms.bf { + return Err(no_v3_domain( + column, + "eql_v3_json carries only ste_vec terms; the other configured \ + indexes would be silently dropped", + "Remove the non-ste_vec indexes from this JSON column or use \ + eqlVersion 2.", + )); + } + if !terms.sv { + return Err(no_v3_domain( + column, + "EQL v3 has no scalar jsonb domain for an index-less JSON column", + "Add a 'ste_vec' index or use eqlVersion 2.", + )); + } + // v3 orders SteVec entries by the CLLW-OPE `op` term under native byte + // comparison. A `Standard`-mode (legacy v2) ste_vec emits + // CLLW-ORE `oc`, whose ciphertext bytes do not order bytewise — + // eql-bindings refuses to convert it rather than silently misorder, and + // no mechanical conversion exists. Catch it here, where the column name + // and a fix are in hand, instead of at encrypt time. + return match terms.sv_mode { + Some(SteVecMode::Compat) => Ok(v3_domain("json")), + _ => Err(no_v3_domain( + column, + "eql_v3_json orders ste_vec entries by the CLLW-OPE 'op' term, but a \ + 'standard' mode ste_vec index emits CLLW-ORE 'oc' terms, which cannot \ + be converted", + "Set the ste_vec index mode to 'compat' (existing rows must be \ + re-encrypted) or use eqlVersion 2.", + )), + }; + } + + if family == "boolean" { + return if terms.any() { + Err(no_v3_domain( + column, + "eql_v3.boolean is storage-only but indexes are configured", + "Remove the indexes or use eqlVersion 2.", + )) + } else { + Ok(v3_domain("boolean")) + }; + } + + // Scalar families, richest capability first. Text ordering domains carry + // hm + ob/op; the non-text ordering domains carry only ob/op. + let is_text = family == "text"; + // cipherstash-config rejects `match` on non-text casts and `ste_vec` on + // non-json casts upstream (into_column_config), but fail closed here too: + // without these guards a non-text `match` + `ore` column would select + // `_ord_ore` and silently drop bf (sv falls through to the fail-closed + // arm at the bottom on its own). + if !is_text && terms.bf { + return Err(no_v3_domain( + column, + format!("eql_v3.{family} domains cannot carry a match/bloom-filter term"), + "Remove the 'match' index (match requires a text cast) or use \ + eqlVersion 2.", + )); + } + // The richest candidate domain and the terms it actually carries. The two + // search domains differ only in their ordering primitive — `_search_ore` + // carries `ob`, `_search` carries `op` — so ORE is preferred when both are + // configured, matching the `_ord_ore` over `_ord_ope` preference below. + let (suffix, carried) = if is_text && terms.hm && terms.ob && terms.bf { + ( + "_search_ore", + ConfiguredTerms { + hm: true, + ob: true, + bf: true, + ..Default::default() + }, + ) + } else if is_text && terms.hm && terms.op && terms.bf { + ( + "_search", + ConfiguredTerms { + hm: true, + op: true, + bf: true, + ..Default::default() + }, + ) + } else if terms.ob && (!is_text || terms.hm) { + ( + "_ord_ore", + ConfiguredTerms { + hm: is_text, + ob: true, + ..Default::default() + }, + ) + } else if terms.op && (!is_text || terms.hm) { + ( + "_ord_ope", + ConfiguredTerms { + hm: is_text, + op: true, + ..Default::default() + }, + ) + } else if is_text && terms.bf { + ( + "_match", + ConfiguredTerms { + bf: true, + ..Default::default() + }, + ) + } else if terms.hm { + ( + "_eq", + ConfiguredTerms { + hm: true, + ..Default::default() + }, + ) + } else if terms.any() { + // Configured terms exist but none of the family's domains can carry + // them (ore/ope-only text, ste_vec on a scalar cast, …). Falling back + // to storage-only would silently drop every configured capability. + return Err(no_v3_domain( + column, + format!("no eql_v3.{family} domain can carry the configured index terms"), + "Ordered text requires a 'unique' index alongside 'ore'/'ope' \ + (v3 text ordering domains carry hm + ob/op). Adjust the indexes \ + or use eqlVersion 2.", + )); + } else { + return Ok(v3_domain(family)); + }; + + // Fail closed if the candidate would DROP a configured capability. + // Coverage is per capability, not per term: equality survives through + // the ORE/OPE operators (so `hm` may drop when `ob`/`op` is carried — + // the documented non-text `unique` + `ore` case), and ordering survives + // when `op` drops in favour of `ob`. `bf` and `sv` have no substitute. + let mut dropped = Vec::new(); + if terms.hm && !(carried.hm || carried.ob || carried.op) { + dropped.push("hm (unique)"); + } + if (terms.ob || terms.op) && !(carried.ob || carried.op) { + if terms.ob { + dropped.push("ob (ore)"); + } + if terms.op { + dropped.push("op (ope)"); + } + } + if terms.bf && !carried.bf { + dropped.push("bf (match)"); + } + if terms.sv { + // Scalar domains never carry sv (the config layer rejects ste_vec on + // non-json casts upstream; fail closed here too). + dropped.push("sv (ste_vec)"); + } + if !dropped.is_empty() { + // Only text combinations can reach this today (non-text bf/sv are + // guarded above and non-text ordering domains cover hm), but keep + // the check generic so new arms stay fail-closed by default. + return Err(no_v3_domain( + column, + format!( + "eql_v3.{family}{suffix} is the closest domain but does not \ + carry the configured {} term(s); stored rows would lose that \ + capability", + dropped.join(", ") + ), + "No single eql_v3 domain covers this index combination — for \ + text, 'unique' + 'ore' + 'match' reaches text_search_ore and \ + 'unique' + 'ope' + 'match' reaches text_search (the richest \ + domains). Adjust the indexes to fit one domain, split \ + the capabilities across separate columns, or use eqlVersion 2.", + )); + } + Ok(v3_domain(&format!("{family}{suffix}"))) +} + +/// A stored payload in whichever wire format the client is configured for. +/// +/// `#[serde(untagged)]` makes the `V2` variant serialize exactly as the bare +/// [`EqlCiphertext`] did before dual-format support (no `Value` round-trip, +/// so v2 output is byte-identical), while `V3` carries the typed +/// [`DomainPayload`] for the column's target domain. `DomainPayload` is +/// itself untagged and Serialize-only, so the v3 wire output carries exactly +/// the keys and values the shape-erased [`eql_bindings::from_v2::from_v2`] +/// `Value` did (pinned by +/// `v3_typed_output_serializes_identically_to_the_from_v2_value`; only the +/// meaningless JSON key order differs — schema wire order instead of a +/// `Value`'s alphabetical order). The v2 payload is boxed because it is +/// substantially larger than the other variant (clippy's +/// `large_enum_variant`). +#[derive(Debug, Serialize)] +#[serde(untagged)] +pub(crate) enum EncryptedOutput { + V2(Box), + V3(DomainPayload), +} + +/// Wrap a Store-mode ciphertext in the client's configured wire format: +/// v2 passes through untouched; v3 converts via +/// [`eql_bindings::from_v2::from_v2_typed`] against the domain selected from +/// the column configuration, keeping the strictly parsed [`DomainPayload`]. +pub(crate) fn storage_output( + ciphertext: EqlCiphertext, + eql_version: EqlVersion, + column_config: &ColumnConfig, +) -> Result { + match eql_version { + EqlVersion::V2 => Ok(EncryptedOutput::V2(Box::new(ciphertext))), + EqlVersion::V3 => { + let target = v3_target_for_column(column_config)?; + let v2_value = serde_json::to_value(&ciphertext)?; + Ok(EncryptedOutput::V3(from_v2_typed(&v2_value, target)?)) + } + } +} + +/// A query payload in whichever wire format the client is configured for. +/// Same untagged pass-through (and boxing) design as [`EncryptedOutput`]. +/// `V3` carries the typed [`QueryPayload`] — a term-only scalar operand +/// (`{v, i, }`, no `c`) for the column domain's `eql_v3.query_` +/// twin, or the `eql_v3.query_jsonb` containment needle. `V3Selector` carries +/// the bare selector hash for `ste_vec_selector` queries — v3 has no +/// encrypted-selector envelope; the SQL `->`/`->>` operators take the +/// [`Selector`] encoding (a string) as `text`. All variants are +/// `#[serde(untagged)]` Serialize-only, so the wire output is exactly the +/// inner value's (keys in schema wire order rather than alphabetical; +/// meaningless for jsonb). +#[derive(Debug, Serialize)] +#[serde(untagged)] +pub(crate) enum QueryOutput { + V2(Box), + V3(QueryPayload), + V3Selector(Selector), +} + +/// Wrap an encrypt-query result in the client's configured wire format. +/// +/// Under v3 every Store-mode result — the scalar full envelope produced for +/// Default queries and the SteVec `sv` document produced for containment — +/// converts through the ONE seam [`from_v2_query_typed`], targeting the +/// column's domain: scalars hoist exactly the domain's required terms into +/// the `{v, i, }` operand (dropping `c`/`k` — the whole point: query +/// operands must never carry a decryptable ciphertext), and ste_vec columns +/// produce the containment needle (stripping the envelope and per-entry +/// ciphertexts, exactly like the SQL cast eql_v3.to_ste_vec_query). The only +/// Query-mode payload with a v3 meaning is the selector, which flattens to +/// its bare selector hash. +pub(crate) fn query_output( + output: EqlOutput, + eql_version: EqlVersion, + column_config: &ColumnConfig, +) -> Result { + match eql_version { + EqlVersion::V2 => Ok(QueryOutput::V2(Box::new(output))), + EqlVersion::V3 => match output { + EqlOutput::Store(ciphertext) => { + let v2_value = serde_json::to_value(&ciphertext)?; + Ok(QueryOutput::V3(from_v2_query_typed( + &v2_value, + v3_target_for_column(column_config)?, + )?)) + } + EqlOutput::Query(EqlQueryPayload::SteVec(payload)) => match payload.term { + SteVecQueryTerm::Selector { selector } => { + Ok(QueryOutput::V3Selector(Selector(selector))) + } + // QueryMode(SteVecSelector) is the only ste_vec query op + // to_query_plaintext leaves in Query mode; hm/oc/containment + // terms arrive via Store mode. + _ => Err(Error::InvariantViolation( + "ste_vec query encryption produced a non-selector term".to_string(), + )), + }, + // Under v3, scalar Default queries run Store mode (the operand + // needs ALL the column domain's terms, not one RootQueryTerm). + EqlOutput::Query(EqlQueryPayload::Encrypted(_)) => Err(Error::InvariantViolation( + "scalar query encryption ran in query mode under eqlVersion 3".to_string(), + )), + }, + } +} + +/// Decode a stored ciphertext value in EITHER wire format into the record + +/// lock context pair zerokms decrypts. +/// +/// Probes the v3 envelope FIRST, then falls back to the typed v2 +/// [`EqlCiphertext`] parse (the historical shape). The order matters: a v3 +/// SteVec document carries both `v: 3` and `k: "sv"`, and the v2 parse is +/// internally tagged on `k` without pinning `v`, so attempted first it would +/// mis-accept the document as a v2 SteVec payload. [`is_v3_payload`] requires +/// `v == 3` exactly, so no v2 payload can take the v3 branch. Decrypt is +/// deliberately version-agnostic — it must keep working across data +/// migrations regardless of the client's `eqlVersion` setting. +pub(crate) fn encrypted_record_from_value( + value: serde_json::Value, + encryption_context: Vec, +) -> Result, Error> { + if is_v3_payload(&value) { + return Ok(WithContext { + record: v3_root_record(&value)?, + context: Cow::Owned(encryption_context), + }); + } + // Not v3 — a parse failure here reports the v2 shape (the shape the + // overwhelming majority of stored data still has). + let ciphertext = EqlCiphertext::deserialize(&value).map_err(Error::Parse)?; + crate::encrypted_record_from_mp_base85(ciphertext, encryption_context) +} + +/// Extract the record ciphertext from a v3 stored payload. +/// +/// Scalars keep the mp_base85 record at the top-level `c`; SteVec documents +/// carry it on the FIRST `sv` entry (`sv[0].c`, the root-selector entry — +/// same invariant as v2, see `encrypted_record_from_mp_base85`). +/// +/// The scalar arm reads `c` directly instead of parsing one of the ~40 +/// generated domain structs: decrypt receives a bare ciphertext with no +/// column configuration, so the specific domain cannot be known here, and +/// the only field decryption needs is `c` (already shape-checked by +/// [`is_v3_payload`]). The structured SteVec arm goes through the typed +/// [`SteVecDocument`] so entry structure is validated before we trust +/// `sv[0]`. +fn v3_root_record(value: &serde_json::Value) -> Result { + if let Some(c) = value.get("c").and_then(serde_json::Value::as_str) { + return EncryptedRecord::from_mp_base85(c).map_err(Error::from); + } + let document = SteVecDocument::deserialize(value).map_err(Error::Parse)?; + let root = document.sv.first().ok_or_else(|| { + Error::InvariantViolation("Missing root entry in v3 SteVec payload".to_string()) + })?; + EncryptedRecord::from_mp_base85(&root.c.0).map_err(Error::from) +} + +/// True when `value` is a stored EQL payload in either wire format. +/// +/// v2 is the strict round-trip through [`EqlCiphertext`]; v3 is the lenient +/// envelope probe [`is_v3_payload`] (`{v: 3, i, c|sv}`). Query payloads — +/// including the v3 containment needle `{sv: […]}` — are not stored payloads +/// and return false. +pub(crate) fn is_encrypted_value(value: &serde_json::Value) -> bool { + EqlCiphertext::deserialize(value).is_ok() || is_v3_payload(value) +} + +/// Resolve the column's v3 domain name against the eql-bindings inventory. +/// +/// [`target_domain_for_column`] only ever emits inventory names (pinned by a +/// unit test), so a parse failure here is a protect-ffi bug, not user error. +fn v3_target_for_column(column_config: &ColumnConfig) -> Result { + let domain = target_domain_for_column(column_config)?; + TargetDomain::parse(&domain).map_err(|e| { + Error::InvariantViolation(format!( + "selected v3 domain {domain:?} is not in the eql-bindings inventory: {e}" + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Shared payload builders for the conversion tests. These mirror the + /// real wire shapes cipherstash-client emits (dummy key material, valid + /// structure) — no mocking of the conversion path itself. + mod support { + use cipherstash_client::eql::{ + EncryptedPayload, EqlCiphertext, Identifier as EqlIdentifier, SteVecEntry, + SteVecEntryTerm, SteVecPayload, EQL_SCHEMA_VERSION, + }; + use cipherstash_client::schema::column::{ColumnMode, ColumnType, Index}; + use cipherstash_client::schema::ColumnConfig; + use cipherstash_client::zerokms::EncryptedRecord; + + pub(super) fn dummy_encrypted_record() -> EncryptedRecord { + EncryptedRecord { + iv: Default::default(), + ciphertext: vec![1; 16], + tag: vec![2; 16], + descriptor: "users/email".to_string(), + keyset_id: None, + decryption_policy: None, + } + } + + pub(super) fn scalar_payload( + hm: Option<&str>, + bf: Option>, + ob: Option>, + ) -> EqlCiphertext { + EqlCiphertext::Encrypted(EncryptedPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "email"), + ciphertext: dummy_encrypted_record(), + hmac_256: hm.map(String::from), + bloom_filter: bf, + ore_block_u64_8_256: ob + .map(|blocks| blocks.into_iter().map(String::from).collect()), + ope_cllw: None, + }) + } + + /// A scalar payload carrying only the `op` (CLLW-OPE) term, as + /// cipherstash-client 0.38.1 emits for an ope-indexed column. + pub(super) fn ope_scalar_payload(op: &str) -> EqlCiphertext { + EqlCiphertext::Encrypted(EncryptedPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "email"), + ciphertext: dummy_encrypted_record(), + hmac_256: None, + bloom_filter: None, + ore_block_u64_8_256: None, + ope_cllw: Some(op.to_string()), + }) + } + + pub(super) fn ste_vec_payload() -> EqlCiphertext { + EqlCiphertext::SteVec(SteVecPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "profile"), + ste_vec: vec![ + SteVecEntry { + selector: "root".into(), + ciphertext: dummy_encrypted_record(), + is_array: None, + term: SteVecEntryTerm::Hmac { + hmac_256: "feedface".into(), + }, + }, + SteVecEntry { + selector: "leaf".into(), + ciphertext: dummy_encrypted_record(), + is_array: Some(true), + // CLLW-OPE: the only sv ordering term v3 can carry. + term: SteVecEntryTerm::Ope { + ope_cllw: "deadbeef".into(), + }, + }, + ], + }) + } + + /// The same document with a CLLW-ORE (`oc`) ordering term, as a + /// `standard`-mode ste_vec index emits. Unconvertible to v3. + pub(super) fn ore_ste_vec_payload() -> EqlCiphertext { + EqlCiphertext::SteVec(SteVecPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "profile"), + ste_vec: vec![SteVecEntry { + selector: "leaf".into(), + ciphertext: dummy_encrypted_record(), + is_array: Some(true), + term: SteVecEntryTerm::OreCllw { + ore_cllw_8: "deadbeef".into(), + }, + }], + }) + } + + pub(super) fn column(cast_type: ColumnType, indexes: Vec) -> ColumnConfig { + ColumnConfig { + name: "test_column".to_string(), + cast_type, + indexes, + in_place: false, + mode: ColumnMode::Encrypted, + } + } + } + + mod validate_eql_version { + use super::*; + + #[test] + fn discriminants_are_the_wire_versions() { + // The enum discriminants double as the JS-facing `eqlVersion` + // values and the on-the-wire `v` — they must never drift. + assert_eq!(EqlVersion::V2 as u8, 2); + assert_eq!(EqlVersion::V3 as u8, 3); + } + + #[test] + fn defaults_to_v2_when_absent() { + assert_eq!(validate_eql_version(None).unwrap(), EqlVersion::V2); + } + + #[test] + fn accepts_v2() { + assert_eq!(validate_eql_version(Some(2)).unwrap(), EqlVersion::V2); + } + + #[test] + fn accepts_v3() { + assert_eq!(validate_eql_version(Some(3)).unwrap(), EqlVersion::V3); + } + + #[test] + fn rejects_other_versions() { + for v in [0u8, 1, 4, 255] { + let err = validate_eql_version(Some(v)).unwrap_err(); + assert!( + err.to_string().contains("eqlVersion"), + "error should mention eqlVersion: {err}" + ); + } + } + } + + mod target_domain_for_column { + use super::*; + use cipherstash_client::schema::column::{ColumnMode, Index, Tokenizer}; + use eql_bindings::from_v2::TargetDomain; + + fn column(cast_type: ColumnType, indexes: Vec) -> ColumnConfig { + ColumnConfig { + name: "test_column".to_string(), + cast_type, + indexes, + in_place: false, + mode: ColumnMode::Encrypted, + } + } + + fn unique() -> Index { + Index::new(IndexType::Unique { + token_filters: vec![], + }) + } + + fn ore() -> Index { + Index::new(IndexType::Ore) + } + + fn ope() -> Index { + Index::new(IndexType::Ope) + } + + fn match_index() -> Index { + Index::new(IndexType::Match { + tokenizer: Tokenizer::Standard, + token_filters: vec![], + k: 6, + m: 2048, + include_original: false, + }) + } + + fn ste_vec() -> Index { + ste_vec_with_mode(SteVecMode::Compat) + } + + fn ste_vec_with_mode(mode: SteVecMode) -> Index { + Index::new(IndexType::SteVec { + prefix: "t/c".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode, + }) + } + + /// The selected domain with the `eql_v3_` typname prefix stripped, so + /// the cases below assert the family/suffix SELECTION and nothing + /// else. That every name carries the prefix is pinned separately by + /// [`selected_domains_carry_the_public_typname_prefix`], and that the + /// prefixed names resolve upstream by + /// [`every_selected_domain_resolves_in_the_v3_inventory`]. + fn domain(cast_type: ColumnType, indexes: Vec) -> String { + let selected = target_domain_for_column(&column(cast_type, indexes)).unwrap(); + selected + .strip_prefix(PUBLIC_TYPNAME_PREFIX) + .unwrap_or_else(|| { + panic!("domain {selected:?} lacks the {PUBLIC_TYPNAME_PREFIX} prefix") + }) + .to_string() + } + + fn domain_err(cast_type: ColumnType, indexes: Vec) -> String { + target_domain_for_column(&column(cast_type, indexes)) + .unwrap_err() + .to_string() + } + + #[test] + fn text_without_indexes_is_storage_only() { + assert_eq!(domain(ColumnType::Text, vec![]), "text"); + } + + #[test] + fn text_unique_is_eq() { + assert_eq!(domain(ColumnType::Text, vec![unique()]), "text_eq"); + } + + #[test] + fn text_match_is_match() { + assert_eq!(domain(ColumnType::Text, vec![match_index()]), "text_match"); + } + + #[test] + fn text_unique_and_ore_is_ord_ore() { + assert_eq!( + domain(ColumnType::Text, vec![unique(), ore()]), + "text_ord_ore" + ); + } + + #[test] + fn text_unique_and_ope_is_ord_ope() { + assert_eq!( + domain(ColumnType::Text, vec![unique(), ope()]), + "text_ord_ope" + ); + } + + #[test] + fn text_unique_ore_match_is_search_ore() { + // The ORE search domain is the `_ore`-suffixed one; the bare + // `text_search` carries the OPE `op` term instead. + assert_eq!( + domain(ColumnType::Text, vec![unique(), ore(), match_index()]), + "text_search_ore" + ); + } + + #[test] + fn text_unique_ope_match_is_search() { + assert_eq!( + domain(ColumnType::Text, vec![unique(), ope(), match_index()]), + "text_search" + ); + } + + #[test] + fn text_unique_ore_ope_match_prefers_search_ore() { + // Both ordering terms configured: ORE wins, mirroring the + // `_ord_ore` over `_ord_ope` preference. Dropping `op` is allowed + // because the ordering capability survives through `ob`. + assert_eq!( + domain( + ColumnType::Text, + vec![unique(), ore(), ope(), match_index()] + ), + "text_search_ore" + ); + } + + #[test] + fn text_unique_ore_and_ope_is_ord_ore() { + // text_ord_ore carries hm + ob. The op term is dropped, but the + // ordering capability survives through ob, so this is an allowed + // drop (mirrors the non-text ore-over-ope preference). + assert_eq!( + domain(ColumnType::Text, vec![unique(), ore(), ope()]), + "text_ord_ore" + ); + } + + #[test] + fn text_unique_and_match_is_an_error() { + // The closest domain, text_match, carries only bf — selecting it + // would permanently strip the configured hm term from stored + // rows. Fail closed instead of silently dropping equality. + let err = domain_err(ColumnType::Text, vec![unique(), match_index()]); + assert!(err.contains("test_column"), "names the column: {err}"); + assert!(err.contains("hm"), "names the dropped term: {err}"); + assert!(err.contains("eqlVersion 2"), "offers a way out: {err}"); + } + + #[test] + fn text_ore_and_match_without_unique_is_an_error() { + // text_match carries only bf; the configured ordering capability + // (ob) would be silently dropped. + let err = domain_err(ColumnType::Text, vec![match_index(), ore()]); + assert!(err.contains("test_column"), "names the column: {err}"); + assert!(err.contains("ob"), "names the dropped term: {err}"); + } + + #[test] + fn text_ore_without_unique_is_an_error() { + // eql_v3.text_ord_ore requires hm + ob; an ore-only text column + // yields no hm, and falling back to storage-only would silently + // drop the configured ordering capability. + let err = domain_err(ColumnType::Text, vec![ore()]); + assert!(err.contains("test_column"), "names the column: {err}"); + assert!(err.contains("unique"), "hints at adding unique: {err}"); + } + + #[test] + fn text_ope_without_unique_is_an_error() { + let err = domain_err(ColumnType::Text, vec![ope()]); + assert!(err.contains("unique"), "hints at adding unique: {err}"); + } + + #[test] + fn int_without_indexes_is_storage_only() { + assert_eq!(domain(ColumnType::Int, vec![]), "integer"); + } + + #[test] + fn int_unique_is_eq() { + assert_eq!(domain(ColumnType::Int, vec![unique()]), "integer_eq"); + } + + #[test] + fn int_ore_is_ord_ore() { + assert_eq!(domain(ColumnType::Int, vec![ore()]), "integer_ord_ore"); + } + + #[test] + fn int_ope_is_ord_ope() { + assert_eq!(domain(ColumnType::Int, vec![ope()]), "integer_ord_ope"); + } + + #[test] + fn int_unique_and_ore_prefers_ord_ore_over_eq() { + // integer_ord_ore requires only ob; hm is dropped but equality + // remains available via the ORE operators (= <>). + assert_eq!( + domain(ColumnType::Int, vec![unique(), ore()]), + "integer_ord_ore" + ); + } + + #[test] + fn int_unique_and_ope_prefers_ord_ope_over_eq() { + assert_eq!( + domain(ColumnType::Int, vec![unique(), ope()]), + "integer_ord_ope" + ); + } + + #[test] + fn int_ore_and_ope_prefers_ord_ore() { + assert_eq!( + domain(ColumnType::Int, vec![ore(), ope()]), + "integer_ord_ore" + ); + } + + #[test] + fn small_int_maps_to_smallint_family() { + assert_eq!( + domain(ColumnType::SmallInt, vec![ore()]), + "smallint_ord_ore" + ); + } + + #[test] + fn big_int_maps_to_bigint_family() { + assert_eq!(domain(ColumnType::BigInt, vec![unique()]), "bigint_eq"); + } + + #[test] + fn big_int_without_indexes_is_storage_only() { + assert_eq!(domain(ColumnType::BigInt, vec![]), "bigint"); + } + + #[test] + fn big_int_ore_is_ord_ore() { + assert_eq!(domain(ColumnType::BigInt, vec![ore()]), "bigint_ord_ore"); + } + + #[test] + fn big_int_unique_and_ore_prefers_ord_ore_over_eq() { + // Same non-text rule as integer: bigint_ord_ore carries only ob; + // hm drops but equality survives via the ORE operators. + assert_eq!( + domain(ColumnType::BigInt, vec![unique(), ore()]), + "bigint_ord_ore" + ); + } + + #[test] + fn big_int_ope_is_ord_ope() { + assert_eq!(domain(ColumnType::BigInt, vec![ope()]), "bigint_ord_ope"); + } + + #[test] + fn float_maps_to_double_family() { + assert_eq!(domain(ColumnType::Float, vec![ore()]), "double_ord_ore"); + } + + #[test] + fn decimal_maps_to_numeric_family() { + assert_eq!(domain(ColumnType::Decimal, vec![ore()]), "numeric_ord_ore"); + } + + #[test] + fn date_maps_to_date_family() { + assert_eq!(domain(ColumnType::Date, vec![ore()]), "date_ord_ore"); + } + + #[test] + fn timestamp_maps_to_timestamp_family() { + assert_eq!( + domain(ColumnType::Timestamp, vec![unique()]), + "timestamp_eq" + ); + } + + #[test] + fn boolean_without_indexes_is_storage_only() { + assert_eq!(domain(ColumnType::Boolean, vec![]), "boolean"); + } + + #[test] + fn boolean_with_unique_is_an_error() { + // eql_v3.boolean is storage-only; any index term would be dropped. + let err = domain_err(ColumnType::Boolean, vec![unique()]); + assert!(err.contains("test_column"), "names the column: {err}"); + assert!(err.contains("storage-only"), "explains bool: {err}"); + } + + #[test] + fn boolean_with_ore_is_an_error() { + let err = domain_err(ColumnType::Boolean, vec![ore()]); + assert!(err.contains("storage-only"), "explains bool: {err}"); + } + + #[test] + fn json_with_compat_mode_ste_vec_is_json() { + assert_eq!(domain(ColumnType::Json, vec![ste_vec()]), "json"); + } + + #[test] + fn json_with_standard_mode_ste_vec_is_an_error() { + // `standard` (the legacy v2 mode) emits CLLW-ORE `oc` + // sv terms. v3 orders sv entries by the CLLW-OPE `op` term under + // native byte comparison, and ORE ciphertext bytes do not order + // that way — converting would silently misorder every entry, so + // eql-bindings refuses. Fail at config time, not encrypt time. + let err = domain_err( + ColumnType::Json, + vec![ste_vec_with_mode(SteVecMode::Standard)], + ); + assert!(err.contains("test_column"), "names the column: {err}"); + assert!(err.contains("compat"), "names the fix: {err}"); + } + + #[test] + fn ste_vec_mode_default_is_compat_so_json_v3_works_unconfigured() { + // cipherstash-config 0.40.0 flipped this default from `standard` + // (CLLW-ORE) to `compat` (CLLW-OPE) — the mode v3 requires. A + // JSON column that names no mode therefore converts. If the + // default ever flips back, the guard above turns v3 JSON into a + // config error, so pin it here rather than discover it downstream. + assert_eq!(SteVecMode::default(), SteVecMode::Compat); + assert_eq!( + domain( + ColumnType::Json, + vec![ste_vec_with_mode(Default::default())] + ), + "json" + ); + } + + #[test] + fn json_without_ste_vec_is_an_error() { + // v2 stores index-less JSON as an opaque scalar (k: "ct"); v3 + // has no scalar jsonb domain to hold it. + let err = domain_err(ColumnType::Json, vec![]); + assert!(err.contains("ste_vec"), "hints at ste_vec: {err}"); + } + + #[test] + fn json_with_ste_vec_and_unique_is_an_error() { + // Upstream config accepts unique/ore/ope alongside ste_vec on a + // JSON column; eql_v3.json carries only sv, so selecting it + // would silently drop the other configured terms. Fail closed. + let err = domain_err(ColumnType::Json, vec![ste_vec(), unique()]); + assert!( + err.contains("ste_vec"), + "explains the sv-only domain: {err}" + ); + } + + #[test] + fn ste_vec_on_a_non_json_cast_is_an_error() { + // The config layer rejects this before it reaches us; fail + // closed anyway rather than silently dropping sv. + let err = domain_err(ColumnType::Int, vec![ste_vec()]); + assert!(err.contains("test_column"), "names the column: {err}"); + } + + #[test] + fn match_mixed_with_ore_on_non_text_is_an_error() { + // Without this guard the _ord_ore arm would match first and + // silently drop bf (the config layer rejects match on non-text + // upstream; fail closed anyway). + let err = domain_err(ColumnType::Int, vec![ore(), match_index()]); + assert!(err.contains("test_column"), "names the column: {err}"); + } + + #[test] + fn match_on_non_text_is_an_error() { + // The config layer rejects this before it reaches us; fail + // closed anyway rather than silently dropping bf. + let err = domain_err(ColumnType::Int, vec![match_index()]); + assert!(err.contains("test_column"), "names the column: {err}"); + } + + #[test] + fn every_selected_domain_resolves_in_the_v3_inventory() { + // The names this function emits must always parse against the + // catalog-generated inventory — a typo here would only surface + // at encrypt time otherwise. + let cases: Vec<(ColumnType, Vec)> = vec![ + (ColumnType::Text, vec![]), + (ColumnType::Text, vec![unique()]), + (ColumnType::Text, vec![match_index()]), + (ColumnType::Text, vec![unique(), ore()]), + (ColumnType::Text, vec![unique(), ope()]), + (ColumnType::Text, vec![unique(), ore(), match_index()]), + (ColumnType::SmallInt, vec![]), + (ColumnType::SmallInt, vec![unique()]), + (ColumnType::SmallInt, vec![ore()]), + (ColumnType::SmallInt, vec![ope()]), + (ColumnType::Int, vec![ore()]), + (ColumnType::BigInt, vec![]), + (ColumnType::BigInt, vec![unique()]), + (ColumnType::BigInt, vec![ore()]), + (ColumnType::BigInt, vec![ope()]), + (ColumnType::Float, vec![ore()]), + (ColumnType::Decimal, vec![ore()]), + (ColumnType::Date, vec![ore()]), + (ColumnType::Timestamp, vec![ore()]), + (ColumnType::Boolean, vec![]), + (ColumnType::Json, vec![ste_vec()]), + ]; + for (cast_type, indexes) in cases { + // The prefixed name as emitted, NOT the stripped `domain()` + // helper: it is the qualified typname that must resolve. + let name = target_domain_for_column(&column(cast_type, indexes)).unwrap(); + assert!( + TargetDomain::parse(&name).is_ok(), + "domain {name:?} must resolve in the eql-bindings inventory" + ); + } + } + + #[test] + fn selected_domains_carry_the_public_typname_prefix() { + // Every public-schema column domain is versioned (`eql_v3_text_eq`, + // `eql_v3_json`); the query twins eql-bindings derives from them + // are not. Storage-only, suffixed and json domains all take it. + for (cast_type, indexes, expected) in [ + (ColumnType::Text, vec![], "eql_v3_text"), + (ColumnType::Text, vec![unique()], "eql_v3_text_eq"), + (ColumnType::Boolean, vec![], "eql_v3_boolean"), + (ColumnType::Json, vec![ste_vec()], "eql_v3_json"), + ] { + assert_eq!( + target_domain_for_column(&column(cast_type, indexes)).unwrap(), + expected + ); + } + } + } + + mod storage_output { + use super::support::{ + column, ope_scalar_payload, ore_ste_vec_payload, scalar_payload, ste_vec_payload, + }; + use super::*; + use cipherstash_client::schema::column::{Index, IndexType, Tokenizer}; + + fn text_search_column() -> ColumnConfig { + column( + ColumnType::Text, + vec![ + Index::new(IndexType::Unique { + token_filters: vec![], + }), + Index::new(IndexType::Ore), + Index::new(IndexType::Match { + tokenizer: Tokenizer::Standard, + token_filters: vec![], + k: 6, + m: 2048, + include_original: false, + }), + ], + ) + } + + #[test] + fn v2_output_serializes_identically_to_the_bare_ciphertext() { + let ciphertext = scalar_payload(Some("aa"), Some(vec![1, 2]), Some(vec!["bb"])); + let expected = serde_json::to_string(&ciphertext).unwrap(); + + let output = storage_output( + scalar_payload(Some("aa"), Some(vec![1, 2]), Some(vec!["bb"])), + EqlVersion::V2, + &text_search_column(), + ) + .unwrap(); + + assert_eq!(serde_json::to_string(&output).unwrap(), expected); + } + + #[test] + fn v3_typed_output_serializes_identically_to_the_from_v2_value() { + // EncryptedOutput::V3 carries the typed DomainPayload + // (from_v2_typed) instead of the shape-erased Value (from_v2). + // Both are #[serde(untagged)], so the FFI wire output must carry + // exactly the from_v2 keys and values — for a scalar domain and + // for the SteVec document domain. (JSON object key ORDER is the + // one permitted difference: a Value serializes keys + // alphabetically, the typed structs in schema wire order + // `v, i, c, `. Key order carries no meaning in JSON and + // none of this crate's consumers observe it.) + use eql_bindings::from_v2::from_v2; + + let scalar_cfg = text_search_column(); + let scalar_ct = scalar_payload(Some("aa"), Some(vec![1, 2]), Some(vec!["bb"])); + let scalar_v2 = serde_json::to_value(&scalar_ct).unwrap(); + // Derived, not spelled: the point is that the typed and + // shape-erased conversions agree on the SAME domain. + let scalar_target = + TargetDomain::parse(&target_domain_for_column(&scalar_cfg).unwrap()).unwrap(); + + let output = storage_output(scalar_ct, EqlVersion::V3, &scalar_cfg).unwrap(); + assert_eq!( + serde_json::to_value(&output).unwrap(), + from_v2(&scalar_v2, scalar_target).unwrap(), + ); + + let sv_cfg = column( + ColumnType::Json, + vec![Index::new(IndexType::SteVec { + prefix: "users/profile".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode: SteVecMode::Compat, + })], + ); + let sv_v2 = serde_json::to_value(ste_vec_payload()).unwrap(); + + let output = storage_output(ste_vec_payload(), EqlVersion::V3, &sv_cfg).unwrap(); + assert_eq!( + serde_json::to_value(&output).unwrap(), + from_v2(&sv_v2, TargetDomain::Json).unwrap(), + ); + } + + #[test] + fn v3_scalar_output_has_v3_envelope_and_required_terms() { + let output = storage_output( + scalar_payload(Some("aa"), Some(vec![1, 2]), Some(vec!["bb"])), + EqlVersion::V3, + &text_search_column(), + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["v"], 3); + assert!(value.get("k").is_none(), "v3 scalar envelope carries no k"); + assert_eq!(value["i"]["t"], "users"); + assert_eq!(value["i"]["c"], "email"); + assert!(value["c"].is_string(), "ciphertext is copied verbatim"); + assert_eq!(value["hm"], "aa"); + assert_eq!(value["ob"], serde_json::json!(["bb"])); + assert_eq!(value["bf"], serde_json::json!([1, 2])); + } + + #[test] + fn v3_output_drops_terms_the_target_domain_does_not_carry() { + // unique + ore on int maps to integer_ord_ore, which carries only + // ob — hm is dropped (equality stays available via ORE). + let cfg = column( + ColumnType::Int, + vec![ + Index::new(IndexType::Unique { + token_filters: vec![], + }), + Index::new(IndexType::Ore), + ], + ); + let output = storage_output( + scalar_payload(Some("aa"), None, Some(vec!["bb"])), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["ob"], serde_json::json!(["bb"])); + assert!(value.get("hm").is_none(), "hm dropped for integer_ord_ore"); + } + + #[test] + fn v3_bigint_eq_output_carries_hm_only() { + // A unique-indexed bigint column maps to public.bigint_eq: + // v, i, c, hm and nothing else (the domain CHECKs in the + // vendored eql-bindings schemas — EQL release + // eql-3.0.0-alpha.3 — require exactly the family terms + // alongside v/i/c). + let cfg = column( + ColumnType::BigInt, + vec![Index::new(IndexType::Unique { + token_filters: vec![], + })], + ); + let output = + storage_output(scalar_payload(Some("aa"), None, None), EqlVersion::V3, &cfg) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["v"], 3); + assert_eq!(value["hm"], "aa"); + let mut keys: Vec<_> = value.as_object().unwrap().keys().collect(); + keys.sort(); + assert_eq!(keys, ["c", "hm", "i", "v"]); + } + + #[test] + fn v3_bigint_ord_ore_output_carries_ob_and_no_hm() { + // An ore-indexed bigint column maps to eql_v3.bigint_ord_ore: + // ob is the ordering term; hm must NOT appear (non-text + // ordering domains carry no hm). + let cfg = column(ColumnType::BigInt, vec![Index::new(IndexType::Ore)]); + let output = storage_output( + scalar_payload(None, None, Some(vec!["bb"])), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["ob"], serde_json::json!(["bb"])); + assert!(value.get("hm").is_none(), "no hm on bigint_ord_ore"); + let mut keys: Vec<_> = value.as_object().unwrap().keys().collect(); + keys.sort(); + assert_eq!(keys, ["c", "i", "ob", "v"]); + } + + #[test] + fn v3_ope_term_flows_through_to_the_ord_ope_domain() { + // cipherstash-client 0.38.1 emits the scalar `op` (CLLW-OPE) + // term (CIP-3348), so an ope-indexed column can reach its + // _ord_ope domain: v, i, c, op and nothing else. + let cfg = column(ColumnType::Int, vec![Index::new(IndexType::Ope)]); + let output = storage_output(ope_scalar_payload("cc"), EqlVersion::V3, &cfg).unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["v"], 3); + assert!(value["c"].is_string(), "ciphertext is copied verbatim"); + assert_eq!(value["op"], "cc"); + let mut keys: Vec<_> = value.as_object().unwrap().keys().collect(); + keys.sort(); + assert_eq!(keys, ["c", "i", "op", "v"]); + } + + #[test] + fn v3_bloom_filter_upper_half_wraps_to_signed() { + // v2 emits unsigned bit positions; the v3 smallint[] encoding + // reinterprets the upper half (32768..=65535) as negative i16. + let output = storage_output( + scalar_payload(Some("aa"), Some(vec![7, 40000]), Some(vec!["bb"])), + EqlVersion::V3, + &text_search_column(), + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["bf"], serde_json::json!([7, 40000u16 as i16])); + } + + #[test] + fn v3_ste_vec_output_is_a_ste_vec_document_with_order_preserved() { + let cfg = column( + ColumnType::Json, + vec![Index::new(IndexType::SteVec { + prefix: "users/profile".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode: SteVecMode::Compat, + })], + ); + let output = storage_output(ste_vec_payload(), EqlVersion::V3, &cfg).unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["v"], 3); + assert_eq!( + value["k"], "sv", + "SteVec documents keep the k form discriminator" + ); + let sv = value["sv"].as_array().unwrap(); + assert_eq!(sv.len(), 2); + // sv[0] is the decryption root — order must survive conversion. + assert_eq!(sv[0]["s"], "root"); + assert_eq!(sv[0]["hm"], "feedface"); + assert!(sv[0]["c"].is_string()); + assert_eq!(sv[1]["s"], "leaf"); + assert_eq!(sv[1]["op"], "deadbeef"); + assert_eq!(sv[1]["a"], true); + } + + #[test] + fn v3_ste_vec_conversion_fails_closed_on_an_ore_entry_term() { + // Defense in depth: target_domain_for_column already rejects a + // `standard`-mode column, so this payload should be unreachable. + // If an `oc` entry ever does arrive (a mode change with rows + // already written under the old mode), conversion must still + // refuse rather than emit a v3 document ordered by ORE bytes. + let cfg = column( + ColumnType::Json, + vec![Index::new(IndexType::SteVec { + prefix: "users/profile".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode: SteVecMode::Compat, + })], + ); + let err = storage_output(ore_ste_vec_payload(), EqlVersion::V3, &cfg) + .unwrap_err() + .to_string(); + assert!( + err.starts_with("EQL v3 conversion failed"), + "carries the conversion-failure prefix: {err}" + ); + assert!(err.contains("re-encrypt"), "names the remedy: {err}"); + } + + #[test] + fn v3_conversion_fails_closed_when_a_required_term_is_missing() { + // text_search_ore requires hm + ob + bf; a payload missing bf must + // not silently degrade. + let result = storage_output( + scalar_payload(Some("aa"), None, Some(vec!["bb"])), + EqlVersion::V3, + &text_search_column(), + ); + + let err = result.unwrap_err().to_string(); + assert!(err.contains("bf"), "names the missing term: {err}"); + // Stable prefix: the TS side maps it to EQL_V3_CONVERSION_FAILED. + assert!( + err.starts_with("EQL v3 conversion failed"), + "carries the conversion-failure prefix: {err}" + ); + } + } + + mod query_output { + use super::support::{column, ope_scalar_payload, scalar_payload, ste_vec_payload}; + use super::*; + use cipherstash_client::eql::{ + EncryptedQueryPayload, EqlOutput, EqlQueryPayload, Identifier as EqlIdentifier, + RootQueryTerm, SteVecQueryPayload, SteVecQueryTerm, EQL_SCHEMA_VERSION, + }; + use cipherstash_client::schema::column::{Index, IndexType, Tokenizer}; + + fn scalar_query() -> EqlOutput { + EqlOutput::Query(EqlQueryPayload::Encrypted(EncryptedQueryPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "email"), + term: RootQueryTerm::Hmac { + hmac_256: "aa".into(), + }, + })) + } + + fn selector_query() -> EqlOutput { + EqlOutput::Query(EqlQueryPayload::SteVec(SteVecQueryPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "profile"), + term: SteVecQueryTerm::Selector { + selector: "deadbeef".into(), + }, + })) + } + + fn text_search_column() -> ColumnConfig { + column( + ColumnType::Text, + vec![ + Index::new(IndexType::Unique { + token_filters: vec![], + }), + Index::new(IndexType::Ore), + Index::new(IndexType::Match { + tokenizer: Tokenizer::Standard, + token_filters: vec![], + k: 6, + m: 2048, + include_original: false, + }), + ], + ) + } + + fn json_column() -> ColumnConfig { + column( + ColumnType::Json, + vec![Index::new(IndexType::SteVec { + prefix: "users/profile".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode: SteVecMode::Compat, + })], + ) + } + + fn sorted_keys(value: &serde_json::Value) -> Vec<&String> { + let mut keys: Vec<_> = value.as_object().unwrap().keys().collect(); + keys.sort(); + keys + } + + #[test] + fn v2_output_serializes_identically_to_the_bare_eql_output() { + let expected = serde_json::to_string(&scalar_query()).unwrap(); + + let output = + query_output(scalar_query(), EqlVersion::V2, &text_search_column()).unwrap(); + + assert_eq!(serde_json::to_string(&output).unwrap(), expected); + } + + #[test] + fn v2_containment_output_passes_through() { + let expected = serde_json::to_string(&EqlOutput::Store(ste_vec_payload())).unwrap(); + + let output = query_output( + EqlOutput::Store(ste_vec_payload()), + EqlVersion::V2, + &json_column(), + ) + .unwrap(); + + assert_eq!(serde_json::to_string(&output).unwrap(), expected); + } + + #[test] + fn v3_containment_output_is_a_query_jsonb_needle() { + let output = query_output( + EqlOutput::Store(ste_vec_payload()), + EqlVersion::V3, + &json_column(), + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + // The eql_v3.query_jsonb needle: {sv: [{s, hm|oc}]} — no + // envelope, no per-entry ciphertext or array marker. + assert!(value.get("v").is_none(), "needle carries no envelope"); + assert!(value.get("i").is_none(), "needle carries no identifier"); + assert!(value.get("k").is_none(), "needle carries no k"); + let sv = value["sv"].as_array().unwrap(); + assert_eq!(sv.len(), 2); + assert_eq!(sv[0]["s"], "root"); + assert_eq!(sv[0]["hm"], "feedface"); + assert!(sv[0].get("c").is_none(), "c is stripped from entries"); + assert_eq!(sv[1]["s"], "leaf"); + assert_eq!(sv[1]["op"], "deadbeef"); + assert!(sv[1].get("a").is_none(), "a is stripped from entries"); + } + + #[test] + fn v3_typed_query_output_serializes_identically_to_the_from_v2_query_value() { + // The QueryOutput::V3 wire form must be byte-identical to what + // the shape-erased from_v2_query Value produced — protect-ffi + // serializes it straight across the FFI boundary. Pinned for the + // containment needle AND a scalar term-only operand. + use eql_bindings::from_v2::from_v2_query; + + let ciphertext = ste_vec_payload(); + let v2_value = serde_json::to_value(&ciphertext).unwrap(); + let erased = from_v2_query(&v2_value, TargetDomain::Json).unwrap(); + + let output = + query_output(EqlOutput::Store(ciphertext), EqlVersion::V3, &json_column()).unwrap(); + let typed = serde_json::to_value(&output).unwrap(); + + // Value equality: identical keys and values. Key ORDER differs + // (the typed struct serializes in schema wire order, the erased + // Value alphabetically) — semantically meaningless for jsonb, + // same caveat as the storage-path pin above. + assert_eq!(typed, erased); + + let scalar_ct = scalar_payload(Some("aa"), Some(vec![1, 2]), Some(vec!["bb"])); + let scalar_v2 = serde_json::to_value(&scalar_ct).unwrap(); + // from_v2_query takes the STORED domain and derives the query twin + // (`eql_v3_text_search_ore` → `query_text_search_ore`) itself. + let scalar_target = + TargetDomain::parse(&target_domain_for_column(&text_search_column()).unwrap()) + .unwrap(); + let erased = from_v2_query(&scalar_v2, scalar_target).unwrap(); + + let output = query_output( + EqlOutput::Store(scalar_ct), + EqlVersion::V3, + &text_search_column(), + ) + .unwrap(); + let typed = serde_json::to_value(&output).unwrap(); + + assert_eq!(typed, erased); + } + + #[test] + fn v3_scalar_store_output_is_a_term_only_operand() { + // A scalar Store envelope hoists to the column domain's query + // twin (eql_v3.query_text_search): all the domain's terms, the + // envelope, and — the point of CIP-3423 — NO ciphertext. + let output = query_output( + EqlOutput::Store(scalar_payload( + Some("aa"), + Some(vec![1, 2]), + Some(vec!["bb"]), + )), + EqlVersion::V3, + &text_search_column(), + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["v"], 3); + assert!(value.get("c").is_none(), "query operands carry no c"); + assert!(value.get("k").is_none(), "query operands carry no k"); + assert_eq!(value["i"]["t"], "users"); + assert_eq!(value["i"]["c"], "email"); + assert_eq!(value["hm"], "aa"); + assert_eq!(value["ob"], serde_json::json!(["bb"])); + assert_eq!(value["bf"], serde_json::json!([1, 2])); + assert_eq!(sorted_keys(&value), ["bf", "hm", "i", "ob", "v"]); + } + + #[test] + fn v3_text_eq_operand_carries_hm_only() { + // unique-only text column → eql_v3.query_text_eq: {v, i, hm}. + let cfg = column( + ColumnType::Text, + vec![Index::new(IndexType::Unique { + token_filters: vec![], + })], + ); + let output = query_output( + EqlOutput::Store(scalar_payload(Some("aa"), None, None)), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["hm"], "aa"); + assert_eq!(sorted_keys(&value), ["hm", "i", "v"]); + } + + #[test] + fn v3_integer_ord_ore_operand_carries_ob_only() { + // ore-indexed int column → eql_v3.query_integer_ord_ore: {v, i, + // ob} (non-text ordering domains carry no hm). + let cfg = column(ColumnType::Int, vec![Index::new(IndexType::Ore)]); + let output = query_output( + EqlOutput::Store(scalar_payload(None, None, Some(vec!["bb"]))), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["ob"], serde_json::json!(["bb"])); + assert_eq!(sorted_keys(&value), ["i", "ob", "v"]); + } + + #[test] + fn v3_integer_ord_ope_operand_carries_op_only() { + // ope-indexed int column → eql_v3.query_integer_ord_ope: {v, i, + // op} (CIP-3348's CLLW-OPE term reaches the query twin too). + let cfg = column(ColumnType::Int, vec![Index::new(IndexType::Ope)]); + let output = query_output( + EqlOutput::Store(ope_scalar_payload("cc")), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["op"], "cc"); + assert_eq!(sorted_keys(&value), ["i", "op", "v"]); + } + + #[test] + fn v3_match_operand_bloom_upper_half_wraps_to_signed() { + // match-only text column → eql_v3.query_text_match: {v, i, bf}, + // with the same u16→i16 reinterpretation as storage conversion. + let cfg = column( + ColumnType::Text, + vec![Index::new(IndexType::Match { + tokenizer: Tokenizer::Standard, + token_filters: vec![], + k: 6, + m: 2048, + include_original: false, + })], + ); + let output = query_output( + EqlOutput::Store(scalar_payload(None, Some(vec![7, 40000]), None)), + EqlVersion::V3, + &cfg, + ) + .unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value["bf"], serde_json::json!([7, 40000u16 as i16])); + assert_eq!(sorted_keys(&value), ["bf", "i", "v"]); + } + + #[test] + fn v3_selector_query_returns_the_bare_selector_string() { + // v3 has no encrypted-selector envelope: the SQL `->`/`->>` + // operators take the bare selector hash as text (the same + // Selector encoding SteVecQuery entries carry in `s`). + let output = query_output(selector_query(), EqlVersion::V3, &json_column()).unwrap(); + + let value = serde_json::to_value(&output).unwrap(); + assert_eq!(value, serde_json::json!("deadbeef")); + } + + #[test] + fn v3_scalar_query_mode_payload_is_an_invariant_violation() { + // Under v3, scalar Default queries run Store mode (the operand + // needs ALL the column domain's terms), so a v2 Query-mode + // scalar payload arriving here means the mode inference broke. + let err = + query_output(scalar_query(), EqlVersion::V3, &text_search_column()).unwrap_err(); + + assert!(matches!(err, Error::InvariantViolation(_))); + assert!( + err.to_string() + .contains("scalar query encryption ran in query mode"), + "names the broken invariant: {err}" + ); + } + + #[test] + fn v3_store_scalar_on_a_json_column_fails_closed() { + // A scalar Store envelope for a ste_vec column targets + // TargetDomain::Json; the conversion rejects the kind mismatch + // instead of emitting a malformed needle. + let err = query_output( + EqlOutput::Store(scalar_payload(Some("aa"), None, None)), + EqlVersion::V3, + &json_column(), + ) + .unwrap_err(); + + let msg = err.to_string(); + // Stable prefix: the TS side maps it to EQL_V3_CONVERSION_FAILED. + assert!( + msg.starts_with("EQL v3 conversion failed"), + "carries the conversion-failure prefix: {msg}" + ); + } + } + + mod dual_format_decrypt { + use super::support::{column, scalar_payload, ste_vec_payload}; + use super::*; + use cipherstash_client::schema::column::{Index, IndexType}; + use serde_json::json; + + fn v2_scalar_value() -> serde_json::Value { + serde_json::to_value(scalar_payload(Some("aa"), None, None)).unwrap() + } + + fn v2_ste_vec_value() -> serde_json::Value { + serde_json::to_value(ste_vec_payload()).unwrap() + } + + fn v3_scalar_value() -> serde_json::Value { + let cfg = column( + ColumnType::Text, + vec![Index::new(IndexType::Unique { + token_filters: vec![], + })], + ); + let output = + storage_output(scalar_payload(Some("aa"), None, None), EqlVersion::V3, &cfg) + .unwrap(); + serde_json::to_value(&output).unwrap() + } + + fn v3_ste_vec_value() -> serde_json::Value { + let cfg = column( + ColumnType::Json, + vec![Index::new(IndexType::SteVec { + prefix: "users/profile".to_string(), + term_filters: vec![], + array_index_mode: Default::default(), + mode: SteVecMode::Compat, + })], + ); + let output = storage_output(ste_vec_payload(), EqlVersion::V3, &cfg).unwrap(); + serde_json::to_value(&output).unwrap() + } + + mod encrypted_record_from_value { + use super::*; + + #[test] + fn decodes_a_v2_scalar_payload() { + let record = encrypted_record_from_value(v2_scalar_value(), vec![]).unwrap(); + assert_eq!(record.record.descriptor, "users/email"); + } + + #[test] + fn decodes_a_v2_ste_vec_payload_from_the_root_entry() { + let record = encrypted_record_from_value(v2_ste_vec_value(), vec![]).unwrap(); + assert_eq!(record.record.descriptor, "users/email"); + } + + #[test] + fn decodes_a_v3_scalar_payload() { + let record = encrypted_record_from_value(v3_scalar_value(), vec![]).unwrap(); + assert_eq!(record.record.descriptor, "users/email"); + } + + #[test] + fn decodes_a_v3_ste_vec_document_from_sv_0() { + let record = encrypted_record_from_value(v3_ste_vec_value(), vec![]).unwrap(); + assert_eq!(record.record.descriptor, "users/email"); + } + + #[test] + fn preserves_the_lock_context() { + let context = vec![zerokms::Context::IdentityClaim("sub".to_string())]; + let record = + encrypted_record_from_value(v3_scalar_value(), context.clone()).unwrap(); + assert_eq!(record.context.len(), 1); + } + + #[test] + fn rejects_plain_json() { + let err = encrypted_record_from_value(json!({"random": "data"}), vec![]); + assert!(err.is_err()); + } + + #[test] + fn rejects_a_v3_document_with_an_empty_sv() { + let value = json!({ + "v": 3, + "k": "sv", + "i": {"t": "users", "c": "profile"}, + "sv": [] + }); + let err = encrypted_record_from_value(value, vec![]).unwrap_err(); + // The v3-specific message also pins ROUTING: the error must + // come from v3_root_record, not the v2 SteVec arm (whose + // message reads "… in SteVec EQL payload"). + assert!( + err.to_string().contains("root entry in v3 SteVec payload"), + "mentions the missing root entry via the v3 branch: {err}" + ); + } + } + + #[test] + fn v2_parse_accepts_a_v3_document_so_v3_must_be_probed_first() { + // A v3 SteVec document carries BOTH `v: 3` and `k: "sv"`. The + // v2 EqlCiphertext parse is internally tagged on `k` and does + // not pin `v`, so it accepts the v3 document as a v2 SteVec + // payload. This canary pins why encrypted_record_from_value + // probes v3 BEFORE attempting the v2 parse — if it ever + // fails, cipherstash-client started rejecting `v: 3` and the + // v3-first ordering became belt-and-braces. + assert!( + EqlCiphertext::deserialize(&v3_ste_vec_value()).is_ok(), + "the v2 parse no longer accepts a v3 SteVec document — \ + cipherstash-client now pins `v`, so the v3-first probe in \ + encrypted_record_from_value is belt-and-braces and this \ + canary can be deleted" + ); + } + + mod is_encrypted_value { + use super::*; + + #[test] + fn accepts_both_wire_formats() { + for value in [ + v2_scalar_value(), + v2_ste_vec_value(), + v3_scalar_value(), + v3_ste_vec_value(), + ] { + assert!(is_encrypted_value(&value), "should accept: {value}"); + } + } + + #[test] + fn rejects_non_payload_values() { + for value in [ + json!({"random": "data"}), + json!("plaintext"), + json!(42), + json!(null), + // v2 envelope without the k discriminator + json!({"v": 2, "i": {"t": "users", "c": "email"}}), + // a v3 containment needle is a QUERY payload, not storage + json!({"sv": [{"s": "aa", "hm": "bb"}]}), + ] { + assert!(!is_encrypted_value(&value), "should reject: {value}"); + } + } + } + } +} diff --git a/crates/protect-ffi-c/src/go_plaintext.rs b/crates/protect-ffi-c/src/go_plaintext.rs index f9cc0e7..93e1398 100644 --- a/crates/protect-ffi-c/src/go_plaintext.rs +++ b/crates/protect-ffi-c/src/go_plaintext.rs @@ -1,13 +1,32 @@ +use chrono::{DateTime, NaiveDate, Utc}; use cipherstash_client::encryption::{Plaintext, TryFromPlaintext, TypeParseError}; use cipherstash_client::schema::ColumnType; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; +/// The plaintext values that cross the FFI boundary from/to Go. +/// +/// Untagged so a bare JSON scalar (string, number, boolean) or object/array +/// deserializes directly. The variant order is load-bearing: `Number` must +/// precede `JsonB` (which accepts any value) so number literals map to +/// `Number`, not the catch-all. +/// +/// `Number` holds a [`serde_json::Number`], NOT an `f64`: the JSON parser +/// preserves integer literals as exact `i64`/`u64`, so a `big_int` value beyond +/// 2^53 survives deserialization losslessly. `to_plaintext_with_type` then +/// takes the exact integer when the target is an integer column, falling back +/// to an exact-or-error `f64` path only for non-integer number forms. +/// +/// Go has no dedicated bigint or date wire form. Large integers arrive as JSON +/// numbers (`Number`); dates/timestamps arrive as RFC 3339 / `YYYY-MM-DD` +/// strings (`String`). On decrypt, integers are emitted as exact JSON integers +/// (via `JsonB`), and dates/timestamps/decimals as strings — see +/// [`TryFrom`]. #[derive(Deserialize, Serialize, Debug, PartialEq)] #[serde(untagged)] pub(crate) enum GoPlaintext { String(String), - Number(f64), + Number(serde_json::Number), Boolean(bool), JsonB(serde_json::Value), } @@ -15,10 +34,14 @@ pub(crate) enum GoPlaintext { impl From<GoPlaintext> for Plaintext { fn from(value: GoPlaintext) -> Self { match value { - GoPlaintext::String(s) => Plaintext::Utf8Str(Some(s)), - GoPlaintext::Number(n) => Plaintext::Float(Some(n)), + GoPlaintext::String(s) => Plaintext::Text(Some(s)), + // Untyped default: a number becomes a float (used only where no + // column cast type is available; typed conversion goes through + // `to_plaintext_with_type`). A `serde_json::Number` always converts + // to `f64`. + GoPlaintext::Number(n) => Plaintext::Float(Some(n.as_f64().unwrap_or(f64::NAN))), GoPlaintext::Boolean(b) => Plaintext::Boolean(Some(b)), - GoPlaintext::JsonB(j) => Plaintext::JsonB(Some(j)), + GoPlaintext::JsonB(j) => Plaintext::Json(Some(j)), } } } @@ -28,93 +51,222 @@ impl TryFrom<Plaintext> for GoPlaintext { fn try_from(value: Plaintext) -> Result<Self, Self::Error> { match value { - v @ Plaintext::Utf8Str(Some(_)) => { - String::try_from_plaintext(v).map(GoPlaintext::String) - } - v @ Plaintext::JsonB(Some(_)) => { + v @ Plaintext::Text(Some(_)) => String::try_from_plaintext(v).map(GoPlaintext::String), + v @ Plaintext::Json(Some(_)) => { serde_json::Value::try_from_plaintext(v).map(GoPlaintext::JsonB) } - Plaintext::BigInt(Some(n)) => Ok(GoPlaintext::Number(n as f64)), - Plaintext::Float(Some(n)) => Ok(GoPlaintext::Number(n)), + // Integer casts decrypt to EXACT JSON integers, never f64 — a lossy + // f64 cast would corrupt values beyond 2^53. Carried in `JsonB` (an + // untagged `serde_json::Number`), which serializes as a bare integer. + Plaintext::BigInt(Some(n)) => Ok(GoPlaintext::JsonB(serde_json::json!(n))), + Plaintext::Int(Some(n)) => Ok(GoPlaintext::JsonB(serde_json::json!(n as i64))), + Plaintext::SmallInt(Some(n)) => Ok(GoPlaintext::JsonB(serde_json::json!(n as i64))), + Plaintext::Float(Some(n)) => serde_json::Number::from_f64(n) + .map(GoPlaintext::Number) + .ok_or_else(|| { + TypeParseError( + "Float value is not representable in JSON (NaN or Infinity)".to_string(), + ) + }), Plaintext::Boolean(Some(b)) => Ok(GoPlaintext::Boolean(b)), + // Decimal is emitted as a JSON string so its full precision survives + // (a JSON number would be re-parsed as f64 on the Go side). + Plaintext::Decimal(Some(d)) => Ok(GoPlaintext::String(d.to_string())), + // Dates and timestamps decrypt to canonical strings. + Plaintext::NaiveDate(Some(nd)) => { + Ok(GoPlaintext::String(nd.format("%Y-%m-%d").to_string())) + } + Plaintext::Timestamp(Some(ts)) => Ok(GoPlaintext::String(ts.to_rfc3339())), _ => Err(TypeParseError("Unsupported type".to_string())), } } } impl GoPlaintext { - /// Convert GoPlaintext to Plaintext based on a target ColumnType. + /// Convert a `GoPlaintext` to a `Plaintext` for the column's storage type. + /// + /// The storage type is driven by `cast_as`, not the input variant. Strings + /// bound for `Date`/`Timestamp` columns are parsed (RFC 3339, plus plain + /// `YYYY-MM-DD` for dates). Numbers bound for integer columns must be + /// represented exactly or the conversion errors — integer literals are + /// taken from the exact `i64`/`u64` the JSON parser preserved (no f64 + /// round-trip), and any lossy cast is rejected rather than silently + /// corrupting the stored value and the index terms derived from it. /// - /// This conversion follows the rule that type coercion is allowed but parsing is not. - /// For example: - /// - GoPlaintext::Number to Plaintext::BigInt is allowed (coercion with truncation) - /// - GoPlaintext::String to Plaintext::BigInt is NOT allowed (would require parsing) + /// Errors never echo the input value: it is plaintext being encrypted. pub fn to_plaintext_with_type( &self, column_type: ColumnType, ) -> Result<Plaintext, TypeParseError> { match (self, column_type) { - // String conversions - only allow to Utf8Str - (GoPlaintext::String(s), ColumnType::Utf8Str) => { - Ok(Plaintext::Utf8Str(Some(s.clone()))) - } - - // Number conversions - allow to numeric types with potential truncation/coercion - (GoPlaintext::Number(n), ColumnType::Float) => Ok(Plaintext::Float(Some(*n))), - (GoPlaintext::Number(n), ColumnType::Decimal) => Decimal::try_from(*n) + // String conversions - Text, Date, and Timestamp (the latter two parse). + (GoPlaintext::String(s), ColumnType::Text) => Ok(Plaintext::Text(Some(s.clone()))), + (GoPlaintext::String(s), ColumnType::Date) => parse_naive_date(s) + .map(|d| Plaintext::NaiveDate(Some(d))) + .map_err(|e| TypeParseError(format!("Cannot parse Date: {}", e))), + (GoPlaintext::String(s), ColumnType::Timestamp) => parse_timestamp(s) + .map(|t| Plaintext::Timestamp(Some(t))) + .map_err(|e| TypeParseError(format!("Cannot parse Timestamp: {}", e))), + + // Float stores the value verbatim (rounding an integer literal to + // f64 is expected for a float column). + (GoPlaintext::Number(n), ColumnType::Float) => n + .as_f64() + .map(|f| Plaintext::Float(Some(f))) + .ok_or_else(|| TypeParseError("Cannot convert number to Float".to_string())), + + // Decimal parses from the number's exact decimal text, NOT via f64, + // so a large integer literal or an exact decimal survives. + (GoPlaintext::Number(n), ColumnType::Decimal) => Decimal::from_str_exact(&n.to_string()) .map(|d| Plaintext::Decimal(Some(d))) - .map_err(|e| TypeParseError(format!("Cannot convert number to Decimal: {}", e))), - (GoPlaintext::Number(n), ColumnType::BigInt) => Ok(Plaintext::BigInt(Some(*n as i64))), - (GoPlaintext::Number(n), ColumnType::Int) => Ok(Plaintext::Int(Some(*n as i32))), + .map_err(|_| { + TypeParseError( + "Cannot convert number to Decimal: value is not representable as a decimal" + .to_string(), + ) + }), + + // Signed integer casts: take the exact i64 first, then range-check; + // fall back to the exact-or-error f64 path for non-integer forms. + (GoPlaintext::Number(n), ColumnType::BigInt) => { + number_to_signed_int::<i64>(n, ColumnType::BigInt).map(|v| Plaintext::BigInt(Some(v))) + } + (GoPlaintext::Number(n), ColumnType::Int) => { + number_to_signed_int::<i32>(n, ColumnType::Int).map(|v| Plaintext::Int(Some(v))) + } (GoPlaintext::Number(n), ColumnType::SmallInt) => { - Ok(Plaintext::SmallInt(Some(*n as i16))) + number_to_signed_int::<i16>(n, ColumnType::SmallInt) + .map(|v| Plaintext::SmallInt(Some(v))) } + + // Unsigned: take the exact u64 first (covers 0..=u64::MAX, including + // values above i64::MAX); negatives error; non-integer forms use the + // exact-or-error f64 path. (GoPlaintext::Number(n), ColumnType::BigUInt) => { - if *n < 0.0 { - Err(TypeParseError( + if let Some(u) = n.as_u64() { + return Ok(Plaintext::BigUInt(Some(u))); + } + let f = n.as_f64().ok_or_else(|| out_of_range(ColumnType::BigUInt))?; + if f < 0.0 { + return Err(TypeParseError( "Cannot convert negative number to BigUInt".to_string(), - )) - } else { - Ok(Plaintext::BigUInt(Some(*n as u64))) + )); } + f64_to_exact_int::<u64>(f, ColumnType::BigUInt).map(|v| Plaintext::BigUInt(Some(v))) } // Boolean conversions - only allow to Boolean (GoPlaintext::Boolean(b), ColumnType::Boolean) => Ok(Plaintext::Boolean(Some(*b))), - // JsonB conversions - only allow to JsonB - (GoPlaintext::JsonB(j), ColumnType::JsonB) => Ok(Plaintext::JsonB(Some(j.clone()))), + // Json conversions - only allow to Json + (GoPlaintext::JsonB(j), ColumnType::Json) => Ok(Plaintext::Json(Some(j.clone()))), - // All other conversions are not allowed - provide helpful error message + // All other conversions are not allowed - provide a helpful message. (go_type, col_type) => { let valid_targets = match go_type { - GoPlaintext::String(_) => "Utf8Str (string columns)", + GoPlaintext::String(_) => { + "Text (string columns), Date, Timestamp (ISO 8601 strings)" + } GoPlaintext::Number(_) => { "Float, BigInt, Int, SmallInt, BigUInt, Decimal (numeric columns)" } GoPlaintext::Boolean(_) => "Boolean (boolean columns)", - GoPlaintext::JsonB(_) => "JsonB (json columns)", + GoPlaintext::JsonB(_) => "Json (json columns)", }; + let type_name = go_plaintext_type_name(go_type); Err(TypeParseError(format!( "Cannot convert {} to {:?}. {} values can only be used with: {}. \ Check your column's cast_as setting in the encrypt config.", - go_plaintext_type_name(go_type), - col_type, - go_plaintext_type_name(go_type), - valid_targets + type_name, col_type, type_name, valid_targets ))) } } } } +/// Error for a number that does not fit the target integer type. Never echoes +/// the value (it is plaintext being encrypted). +fn out_of_range(column_type: ColumnType) -> TypeParseError { + TypeParseError(format!( + "Cannot convert number to {:?}: value is out of range", + column_type + )) +} + +/// Convert a JSON number to a signed integer type exactly. +/// +/// Integer literals are taken from the exact `i64` the JSON parser preserved, +/// so precision beyond 2^53 survives; a value that does not fit the target is +/// rejected. Non-integer (f64-form) numbers fall back to the exact-or-error +/// [`f64_to_exact_int`] path. +fn number_to_signed_int<T>( + n: &serde_json::Number, + column_type: ColumnType, +) -> Result<T, TypeParseError> +where + T: TryFrom<i64> + TryFrom<i128>, +{ + if let Some(i) = n.as_i64() { + return T::try_from(i).map_err(|_| out_of_range(column_type)); + } + let f = n.as_f64().ok_or_else(|| out_of_range(column_type))?; + f64_to_exact_int::<T>(f, column_type) +} + +/// Convert an `f64` into an integer type exactly, or error. +/// +/// A saturating `as` cast would silently corrupt out-of-range values, map NaN +/// to 0, and drop fractional parts — and the index terms would be computed over +/// the corrupted value. Error unless the value is finite, integral, and fits +/// the target. The error deliberately does not echo the value. +fn f64_to_exact_int<T: TryFrom<i128>>( + n: f64, + column_type: ColumnType, +) -> Result<T, TypeParseError> { + if !n.is_finite() { + return Err(TypeParseError(format!( + "Cannot convert number to {:?}: value must be finite (got NaN or Infinity)", + column_type + ))); + } + if n.fract() != 0.0 { + return Err(TypeParseError(format!( + "Cannot convert number to {:?}: value has a fractional component", + column_type + ))); + } + // `n` is finite with no fractional part, so if it lies within i128's range + // the `as` cast below is exact. `i128::MAX as f64` rounds up to 2^127, + // which does NOT fit in i128, hence `>=`. + if n < i128::MIN as f64 || n >= i128::MAX as f64 { + return Err(out_of_range(column_type)); + } + T::try_from(n as i128).map_err(|_| out_of_range(column_type)) +} + +/// Parse a string into a `NaiveDate`. Accepts full RFC 3339 timestamps and +/// plain `YYYY-MM-DD`. +fn parse_naive_date(s: &str) -> Result<NaiveDate, String> { + if let Ok(dt) = DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc).date_naive()); + } + NaiveDate::parse_from_str(s, "%Y-%m-%d").map_err(|e| e.to_string()) +} + +/// Parse a string into a UTC `DateTime`. Accepts RFC 3339. +fn parse_timestamp(s: &str) -> Result<DateTime<Utc>, String> { + DateTime::parse_from_rfc3339(s) + .map(|dt| dt.with_timezone(&Utc)) + .map_err(|e| e.to_string()) +} + /// Helper function to get a readable type name for error messages pub(crate) fn go_plaintext_type_name(go_plaintext: &GoPlaintext) -> &'static str { match go_plaintext { GoPlaintext::String(_) => "String", GoPlaintext::Number(_) => "Number", GoPlaintext::Boolean(_) => "Boolean", - GoPlaintext::JsonB(_) => "JsonB", + GoPlaintext::JsonB(_) => "Json", } } @@ -122,37 +274,50 @@ pub(crate) fn go_plaintext_type_name(go_plaintext: &GoPlaintext) -> &'static str mod tests { use super::*; + /// A `GoPlaintext::Number` from an f64 (fractional/float forms). + fn num(f: f64) -> GoPlaintext { + GoPlaintext::Number(serde_json::Number::from_f64(f).unwrap()) + } + + /// A `GoPlaintext::Number` from an exact i64 integer literal. + fn int(i: i64) -> GoPlaintext { + GoPlaintext::Number(serde_json::Number::from(i)) + } + + fn sample_dt() -> DateTime<Utc> { + DateTime::parse_from_rfc3339("2025-03-14T12:34:56.789Z") + .unwrap() + .with_timezone(&Utc) + } + mod go_plaintext_to_plaintext { use super::*; #[test] fn test_string() { - let go_string = GoPlaintext::String("hello".to_string()); - let plaintext: Plaintext = go_string.into(); - assert_eq!(plaintext, Plaintext::Utf8Str(Some("hello".to_string()))); + let plaintext: Plaintext = GoPlaintext::String("hello".to_string()).into(); + assert_eq!(plaintext, Plaintext::Text(Some("hello".to_string()))); } #[test] fn test_number() { - let go_number = GoPlaintext::Number(42.5); - let plaintext: Plaintext = go_number.into(); + let plaintext: Plaintext = num(42.5).into(); assert_eq!(plaintext, Plaintext::Float(Some(42.5))); } #[test] fn test_boolean() { - let go_bool = GoPlaintext::Boolean(true); - let plaintext: Plaintext = go_bool.into(); + let plaintext: Plaintext = GoPlaintext::Boolean(true).into(); assert_eq!(plaintext, Plaintext::Boolean(Some(true))); } #[test] fn test_jsonb() { - let go_jsonb = GoPlaintext::JsonB(serde_json::json!({"key": "value"})); - let plaintext: Plaintext = go_jsonb.into(); + let plaintext: Plaintext = + GoPlaintext::JsonB(serde_json::json!({"key": "value"})).into(); assert_eq!( plaintext, - Plaintext::JsonB(Some(serde_json::json!({"key": "value"}))) + Plaintext::Json(Some(serde_json::json!({"key": "value"}))) ); } } @@ -161,47 +326,81 @@ mod tests { use super::*; #[test] - fn test_utf8str() { - let plaintext = Plaintext::Utf8Str(Some("hello".to_string())); - let go_plaintext: GoPlaintext = plaintext.try_into().unwrap(); - assert_eq!(go_plaintext, GoPlaintext::String("hello".to_string())); + fn test_text() { + let go: GoPlaintext = Plaintext::Text(Some("hello".to_string())).try_into().unwrap(); + assert_eq!(go, GoPlaintext::String("hello".to_string())); } #[test] fn test_float() { - let plaintext = Plaintext::Float(Some(42.5)); - let go_plaintext: GoPlaintext = plaintext.try_into().unwrap(); - assert_eq!(go_plaintext, GoPlaintext::Number(42.5)); + let go: GoPlaintext = Plaintext::Float(Some(42.5)).try_into().unwrap(); + assert_eq!(go, num(42.5)); } #[test] fn test_boolean() { - let plaintext = Plaintext::Boolean(Some(true)); - let go_plaintext: GoPlaintext = plaintext.try_into().unwrap(); - assert_eq!(go_plaintext, GoPlaintext::Boolean(true)); + let go: GoPlaintext = Plaintext::Boolean(Some(true)).try_into().unwrap(); + assert_eq!(go, GoPlaintext::Boolean(true)); } #[test] - fn test_jsonb() { - let plaintext = Plaintext::JsonB(Some(serde_json::json!({"key": "value"}))); - let go_plaintext: GoPlaintext = plaintext.try_into().unwrap(); - assert_eq!( - go_plaintext, - GoPlaintext::JsonB(serde_json::json!({"key": "value"})) - ); + fn test_json() { + let go: GoPlaintext = Plaintext::Json(Some(serde_json::json!({"key": "value"}))) + .try_into() + .unwrap(); + assert_eq!(go, GoPlaintext::JsonB(serde_json::json!({"key": "value"}))); + } + + #[test] + fn test_bigint_becomes_exact_json_integer() { + for v in [i64::MIN, i64::MAX, 0, -1, 9_007_199_254_740_995] { + let go: GoPlaintext = Plaintext::BigInt(Some(v)).try_into().unwrap(); + assert_eq!(go, GoPlaintext::JsonB(serde_json::json!(v))); + assert_eq!(serde_json::to_string(&go).unwrap(), v.to_string()); + } } #[test] - fn test_bigint() { - let plaintext = Plaintext::BigInt(Some(42)); - let go_plaintext: GoPlaintext = plaintext.try_into().unwrap(); - assert_eq!(go_plaintext, GoPlaintext::Number(42.0)); + fn test_int_and_small_int_become_json_integers() { + let go: GoPlaintext = Plaintext::Int(Some(42)).try_into().unwrap(); + assert_eq!(serde_json::to_string(&go).unwrap(), "42"); + let go: GoPlaintext = Plaintext::SmallInt(Some(-7)).try_into().unwrap(); + assert_eq!(serde_json::to_string(&go).unwrap(), "-7"); + } + + #[test] + fn test_decimal_becomes_json_string() { + let d = Decimal::new(1999, 2); // 19.99 + let go: GoPlaintext = Plaintext::Decimal(Some(d)).try_into().unwrap(); + assert_eq!(go, GoPlaintext::String("19.99".to_string())); + } + + #[test] + fn test_naive_date_becomes_yyyy_mm_dd_string() { + let d = NaiveDate::from_ymd_opt(2025, 3, 14).unwrap(); + let go: GoPlaintext = Plaintext::NaiveDate(Some(d)).try_into().unwrap(); + assert_eq!(go, GoPlaintext::String("2025-03-14".to_string())); + } + + #[test] + fn test_timestamp_becomes_rfc3339_string() { + let t = sample_dt(); + let go: GoPlaintext = Plaintext::Timestamp(Some(t)).try_into().unwrap(); + assert_eq!(go, GoPlaintext::String(t.to_rfc3339())); + } + + #[test] + fn test_float_nan_is_rejected() { + // serde_json cannot represent NaN, so a Float(NaN) decrypt errors + // rather than producing an unserializable value. + let result: Result<GoPlaintext, TypeParseError> = + Plaintext::Float(Some(f64::NAN)).try_into(); + assert!(result.is_err()); } #[test] fn test_unsupported_type_returns_error() { - let plaintext = Plaintext::Utf8Str(None); - let result: Result<GoPlaintext, TypeParseError> = plaintext.try_into(); + let result: Result<GoPlaintext, TypeParseError> = Plaintext::Text(None).try_into(); assert!(result.is_err()); } } @@ -210,167 +409,312 @@ mod tests { use super::*; #[test] - fn test_string_to_utf8str() { - let go_string = GoPlaintext::String("hello".to_string()); - let result = go_string - .to_plaintext_with_type(ColumnType::Utf8Str) + fn test_string_to_text() { + let result = GoPlaintext::String("hello".to_string()) + .to_plaintext_with_type(ColumnType::Text) .unwrap(); - assert_eq!(result, Plaintext::Utf8Str(Some("hello".to_string()))); + assert_eq!(result, Plaintext::Text(Some("hello".to_string()))); } #[test] fn test_string_to_int_fails() { - let go_string = GoPlaintext::String("123".to_string()); - let result = go_string.to_plaintext_with_type(ColumnType::Int); - assert!(result.is_err()); + let result = + GoPlaintext::String("123".to_string()).to_plaintext_with_type(ColumnType::Int); assert!(result.unwrap_err().0.contains("Cannot convert")); } #[test] fn test_number_to_float() { - let go_number = GoPlaintext::Number(3.78); - let result = go_number.to_plaintext_with_type(ColumnType::Float).unwrap(); + let result = num(3.78).to_plaintext_with_type(ColumnType::Float).unwrap(); assert_eq!(result, Plaintext::Float(Some(3.78))); } #[test] - fn test_number_to_decimal() { - let go_number = GoPlaintext::Number(3.78); - let result = go_number - .to_plaintext_with_type(ColumnType::Decimal) + fn test_large_integer_literal_to_float_rounds() { + // A big integer literal into a float column is accepted (f64 rounds). + let result = int(9_007_199_254_740_995) + .to_plaintext_with_type(ColumnType::Float) .unwrap(); - let expected_decimal = Decimal::try_from(3.78).unwrap(); - assert_eq!(result, Plaintext::Decimal(Some(expected_decimal))); + assert_eq!( + result, + Plaintext::Float(Some(9_007_199_254_740_995_i64 as f64)) + ); } + // --- Exact-integer behaviour (the precision regression) --- + #[test] - fn test_number_to_bigint_truncates() { - let go_number = GoPlaintext::Number(42.7); - let result = go_number - .to_plaintext_with_type(ColumnType::BigInt) - .unwrap(); - assert_eq!(result, Plaintext::BigInt(Some(42))); + fn bigint_is_exact_beyond_2_53() { + for v in [9_007_199_254_740_995_i64, i64::MAX, i64::MIN, 0, -1] { + let result = int(v).to_plaintext_with_type(ColumnType::BigInt).unwrap(); + assert_eq!(result, Plaintext::BigInt(Some(v))); + } + } + + #[test] + fn bigint_is_exact_through_json_deserialization() { + // The regression: deserializing the literal must NOT round to f64. + let go: GoPlaintext = serde_json::from_str("9007199254740995").unwrap(); + let result = go.to_plaintext_with_type(ColumnType::BigInt).unwrap(); + assert_eq!(result, Plaintext::BigInt(Some(9_007_199_254_740_995))); } #[test] - fn test_number_to_int_truncates() { - let go_number = GoPlaintext::Number(42.7); - let result = go_number.to_plaintext_with_type(ColumnType::Int).unwrap(); + fn integer_literal_out_of_int_range_fails() { + let err = int(5_000_000_000) + .to_plaintext_with_type(ColumnType::Int) + .unwrap_err(); + assert!(err.0.contains("Int"), "{}", err.0); + assert!(err.0.contains("out of range"), "{}", err.0); + } + + #[test] + fn in_range_integer_literal_to_int() { + let result = int(42).to_plaintext_with_type(ColumnType::Int).unwrap(); assert_eq!(result, Plaintext::Int(Some(42))); + let max = int(i32::MAX as i64) + .to_plaintext_with_type(ColumnType::Int) + .unwrap(); + assert_eq!(max, Plaintext::Int(Some(i32::MAX))); } #[test] - fn test_number_to_smallint_truncates() { - let go_number = GoPlaintext::Number(42.7); - let result = go_number + fn smallint_range_check() { + let ok = int(i16::MAX as i64) .to_plaintext_with_type(ColumnType::SmallInt) .unwrap(); - assert_eq!(result, Plaintext::SmallInt(Some(42))); + assert_eq!(ok, Plaintext::SmallInt(Some(i16::MAX))); + let err = int(40_000) + .to_plaintext_with_type(ColumnType::SmallInt) + .unwrap_err(); + assert!(err.0.contains("SmallInt") && err.0.contains("out of range")); + } + + #[test] + fn biguint_accepts_u64_above_i64_max() { + let v: u64 = (i64::MAX as u64) + 1; + let go = GoPlaintext::Number(serde_json::Number::from(v)); + assert_eq!( + go.to_plaintext_with_type(ColumnType::BigUInt).unwrap(), + Plaintext::BigUInt(Some(v)) + ); + // And exact through the JSON boundary. + let go2: GoPlaintext = serde_json::from_str(&v.to_string()).unwrap(); + assert_eq!( + go2.to_plaintext_with_type(ColumnType::BigUInt).unwrap(), + Plaintext::BigUInt(Some(v)) + ); + } + + #[test] + fn biguint_max_is_exact() { + let go = GoPlaintext::Number(serde_json::Number::from(u64::MAX)); + assert_eq!( + go.to_plaintext_with_type(ColumnType::BigUInt).unwrap(), + Plaintext::BigUInt(Some(u64::MAX)) + ); } #[test] - fn test_number_to_biguint() { - let go_number = GoPlaintext::Number(42.0); - let result = go_number + fn negative_integer_to_biguint_fails() { + let err = int(-42) .to_plaintext_with_type(ColumnType::BigUInt) - .unwrap(); - assert_eq!(result, Plaintext::BigUInt(Some(42))); + .unwrap_err(); + assert!(err.0.contains("negative"), "{}", err.0); } + // --- Fractional / f64-form numbers still rejected for integer casts --- + #[test] - fn test_negative_number_to_biguint_fails() { - let go_number = GoPlaintext::Number(-42.0); - let result = go_number.to_plaintext_with_type(ColumnType::BigUInt); - assert!(result.is_err()); - assert!(result.unwrap_err().0.contains("negative")); + fn fractional_number_to_int_fails() { + let err = num(42.5).to_plaintext_with_type(ColumnType::Int).unwrap_err(); + assert!(err.0.contains("Int"), "{}", err.0); + // Reaches the exact-or-error f64 path, which reports the fractional part. + assert!(err.0.contains("fractional") || err.0.contains("out of range")); + } + + #[test] + fn fractional_number_to_bigint_fails() { + let err = num(42.5) + .to_plaintext_with_type(ColumnType::BigInt) + .unwrap_err(); + assert!(err.0.contains("BigInt"), "{}", err.0); + } + + #[test] + fn f64_form_out_of_range_to_bigint_fails() { + // 2^63 as an f64 literal exceeds i64::MAX. + let err = num(9_223_372_036_854_775_808.0) + .to_plaintext_with_type(ColumnType::BigInt) + .unwrap_err(); + assert!(err.0.contains("out of range"), "{}", err.0); + } + + #[test] + fn negative_fractional_to_biguint_fails() { + let err = num(-42.0) + .to_plaintext_with_type(ColumnType::BigUInt) + .unwrap_err(); + assert!(err.0.contains("negative"), "{}", err.0); + } + + // --- Decimal exactness (no f64 round-trip) --- + + #[test] + fn decimal_from_fractional_literal() { + let go: GoPlaintext = serde_json::from_str("0.1").unwrap(); + match go.to_plaintext_with_type(ColumnType::Decimal).unwrap() { + Plaintext::Decimal(Some(d)) => assert_eq!(d.to_string(), "0.1"), + other => panic!("expected Decimal, got {other:?}"), + } + } + + #[test] + fn decimal_from_large_integer_is_exact() { + // A large integer into a decimal column is exact — via the number's + // decimal text, not f64 (which would round at 2^53). + let go: GoPlaintext = serde_json::from_str("9007199254740995").unwrap(); + match go.to_plaintext_with_type(ColumnType::Decimal).unwrap() { + Plaintext::Decimal(Some(d)) => assert_eq!(d.to_string(), "9007199254740995"), + other => panic!("expected Decimal, got {other:?}"), + } + } + + #[test] + fn errors_do_not_echo_the_value() { + let err = int(5_000_000_001) + .to_plaintext_with_type(ColumnType::Int) + .unwrap_err(); + assert!( + !err.0.contains("5000000001"), + "error must not echo the plaintext value, got: {}", + err.0 + ); } #[test] fn test_boolean_to_boolean() { - let go_bool = GoPlaintext::Boolean(true); - let result = go_bool.to_plaintext_with_type(ColumnType::Boolean).unwrap(); + let result = GoPlaintext::Boolean(true) + .to_plaintext_with_type(ColumnType::Boolean) + .unwrap(); assert_eq!(result, Plaintext::Boolean(Some(true))); } #[test] fn test_boolean_to_string_fails() { - let go_bool = GoPlaintext::Boolean(true); - let result = go_bool.to_plaintext_with_type(ColumnType::Utf8Str); - assert!(result.is_err()); + let result = GoPlaintext::Boolean(true).to_plaintext_with_type(ColumnType::Text); assert!(result.unwrap_err().0.contains("Cannot convert")); } #[test] - fn test_jsonb_to_jsonb() { + fn test_jsonb_to_json() { let json_value = serde_json::json!({"key": "value"}); - let go_jsonb = GoPlaintext::JsonB(json_value.clone()); - let result = go_jsonb.to_plaintext_with_type(ColumnType::JsonB).unwrap(); - assert_eq!(result, Plaintext::JsonB(Some(json_value))); + let result = GoPlaintext::JsonB(json_value.clone()) + .to_plaintext_with_type(ColumnType::Json) + .unwrap(); + assert_eq!(result, Plaintext::Json(Some(json_value))); } #[test] fn test_jsonb_to_string_fails() { - let json_value = serde_json::json!({"key": "value"}); - let go_jsonb = GoPlaintext::JsonB(json_value); - let result = go_jsonb.to_plaintext_with_type(ColumnType::Utf8Str); - assert!(result.is_err()); + let result = GoPlaintext::JsonB(serde_json::json!({"key": "value"})) + .to_plaintext_with_type(ColumnType::Text); assert!(result.unwrap_err().0.contains("Cannot convert")); } #[test] - fn test_number_to_boolean_fails() { - let go_number = GoPlaintext::Number(1.0); - let result = go_number.to_plaintext_with_type(ColumnType::Boolean); - assert!(result.is_err()); - assert!(result.unwrap_err().0.contains("Cannot convert")); + fn test_iso_date_string_to_date() { + let result = GoPlaintext::String("2025-03-14".to_string()) + .to_plaintext_with_type(ColumnType::Date) + .unwrap(); + assert_eq!( + result, + Plaintext::NaiveDate(Some(NaiveDate::from_ymd_opt(2025, 3, 14).unwrap())) + ); } #[test] - fn test_type_coercion_error_shows_valid_alternatives() { - let go_string = GoPlaintext::String("hello".to_string()); - let result = go_string.to_plaintext_with_type(ColumnType::Int); - assert!(result.is_err()); - let err_msg = result.unwrap_err().0; - assert!( - err_msg.contains("Utf8Str"), - "Error should mention valid target Utf8Str: {}", - err_msg - ); - assert!( - err_msg.contains("cast_as"), - "Error should mention cast_as setting: {}", - err_msg + fn test_rfc3339_string_to_date_truncates_time() { + let result = GoPlaintext::String("2025-03-14T12:34:56.789Z".to_string()) + .to_plaintext_with_type(ColumnType::Date) + .unwrap(); + assert_eq!( + result, + Plaintext::NaiveDate(Some(NaiveDate::from_ymd_opt(2025, 3, 14).unwrap())) ); + } - let go_number = GoPlaintext::Number(42.0); - let result = go_number.to_plaintext_with_type(ColumnType::Boolean); - assert!(result.is_err()); - let err_msg = result.unwrap_err().0; - assert!( - err_msg.contains("Float") || err_msg.contains("BigInt"), - "Error should mention valid numeric targets: {}", - err_msg - ); + #[test] + fn test_rfc3339_string_to_timestamp() { + let result = GoPlaintext::String("2025-03-14T12:34:56.789Z".to_string()) + .to_plaintext_with_type(ColumnType::Timestamp) + .unwrap(); + assert_eq!(result, Plaintext::Timestamp(Some(sample_dt()))); + } - let go_bool = GoPlaintext::Boolean(true); - let result = go_bool.to_plaintext_with_type(ColumnType::Int); - assert!(result.is_err()); - let err_msg = result.unwrap_err().0; - assert!( - err_msg.contains("Boolean"), - "Error should mention valid target Boolean: {}", - err_msg - ); + #[test] + fn test_invalid_date_string_fails_without_echoing_input() { + let err = GoPlaintext::String("not a date".to_string()) + .to_plaintext_with_type(ColumnType::Date) + .expect_err("unparseable input must fail"); + assert!(err.0.contains("Cannot parse Date"), "{}", err.0); + assert!(!err.0.contains("not a date"), "{}", err.0); + } - let go_json = GoPlaintext::JsonB(serde_json::json!({"a": 1})); - let result = go_json.to_plaintext_with_type(ColumnType::Int); - assert!(result.is_err()); - let err_msg = result.unwrap_err().0; - assert!( - err_msg.contains("JsonB"), - "Error should mention valid target JsonB: {}", - err_msg - ); + #[test] + fn test_date_only_string_fails_as_timestamp() { + let err = GoPlaintext::String("2025-03-14".to_string()) + .to_plaintext_with_type(ColumnType::Timestamp) + .expect_err("date-only string must fail as Timestamp"); + assert!(err.0.contains("Cannot parse Timestamp")); + assert!(!err.0.contains("2025-03-14")); + } + + #[test] + fn test_type_coercion_error_shows_valid_alternatives() { + let err = GoPlaintext::String("hello".to_string()) + .to_plaintext_with_type(ColumnType::Int) + .unwrap_err() + .0; + assert!(err.contains("Text"), "{}", err); + assert!(err.contains("cast_as"), "{}", err); + + let err = GoPlaintext::JsonB(serde_json::json!({"a": 1})) + .to_plaintext_with_type(ColumnType::Int) + .unwrap_err() + .0; + assert!(err.contains("Json"), "{}", err); + } + } + + mod wire_deserialization { + use super::*; + + #[test] + fn integer_literal_deserializes_as_number_not_jsonb() { + let go: GoPlaintext = serde_json::from_str("42").unwrap(); + assert_eq!(go, GoPlaintext::Number(serde_json::Number::from(42))); + } + + #[test] + fn large_integer_literal_deserializes_exactly() { + let go: GoPlaintext = serde_json::from_str("9007199254740995").unwrap(); + match go { + GoPlaintext::Number(n) => assert_eq!(n.as_i64(), Some(9_007_199_254_740_995)), + other => panic!("expected Number, got {other:?}"), + } + } + + #[test] + fn object_deserializes_as_jsonb() { + let go: GoPlaintext = serde_json::from_str(r#"{"key":"value"}"#).unwrap(); + assert_eq!(go, GoPlaintext::JsonB(serde_json::json!({"key": "value"}))); + } + + #[test] + fn string_deserializes_as_string() { + let go: GoPlaintext = serde_json::from_str(r#""hello""#).unwrap(); + assert_eq!(go, GoPlaintext::String("hello".to_string())); } } } diff --git a/crates/protect-ffi-c/src/lib.rs b/crates/protect-ffi-c/src/lib.rs index b18fd67..d875a2d 100644 --- a/crates/protect-ffi-c/src/lib.rs +++ b/crates/protect-ffi-c/src/lib.rs @@ -3,28 +3,36 @@ // function `unsafe` would change the C header signature, which is undesirable. #![allow(clippy::not_unsafe_ptr_arg_deref)] -mod encrypt_config; +mod auth; +mod eql_v3; mod go_plaintext; +use auth::{ + AuthStrategyType, GoAuthStrategy, GoOidcProvider, GoProvidedTokenStrategy, GoTokenCallback, + ProtectTokenFn, +}; use cipherstash_client::{ - credentials::ServiceToken, encryption::{EncryptionError, Plaintext, QueryOp, ScopedCipher, TypeParseError}, eql::{ - encrypt_eql, EqlCiphertext, EqlEncryptOpts, EqlError, EqlOperation, + encrypt_eql, EqlCiphertext, EqlEncryptOpts, EqlError, EqlOperation, EqlOutput, Identifier as EqlIdentifier, PreparedPlaintext, }, schema::{ column::{Index, IndexType}, - ColumnConfig, + errors::ConfigError, + CanonicalEncryptionConfig, ColumnConfig, Identifier, }, zerokms::{ - self, FallbackKeyProvider, RecordDecryptError, SecretKey, WithContext, ZeroKMSBuilder, - ZeroKMSBuilderError, ZeroKMSWithClientKey, + self, FallbackKeyProvider, RecordDecryptError, SecretKey, WithContext, + ZeroKMSBuilder, ZeroKMSBuilderError, ZeroKMSWithClientKey, }, AuthError, AutoStrategy, IdentifiedBy, UnverifiedContext, }; use cts_common::Crn; -use encrypt_config::{EncryptConfig, Identifier}; +use eql_v3::{ + encrypted_record_from_value, is_encrypted_value, query_output, storage_output, + validate_eql_version, EncryptedOutput, EqlVersion, QueryOutput, +}; use go_plaintext::GoPlaintext; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -62,24 +70,18 @@ impl Default for CResult { // Client // --------------------------------------------------------------------------- -type ScopedZeroKMS = ScopedCipher<AutoStrategy>; +type ScopedZeroKMS = ScopedCipher<GoAuthStrategy>; /// Opaque client handle passed across the FFI boundary. -#[derive(Clone)] pub struct Client { cipher: Arc<ScopedZeroKMS>, - zerokms: Arc<ZeroKMSWithClientKey<AutoStrategy>>, + zerokms: Arc<ZeroKMSWithClientKey<GoAuthStrategy>>, encrypt_config: Arc<HashMap<Identifier, ColumnConfig>>, + /// EQL wire version this client emits. Decryption accepts both formats + /// regardless of this setting. + eql_version: EqlVersion, } -/// Re-export EqlCiphertext as Encrypted for backward compatibility. -/// -/// This is a unified structure that contains the identifier, version, and the encrypted body -/// with all associated cryptographic searchable encrypted metadata (SEM). -/// -/// Note: The ciphertext field (c) is serialized in MessagePack Base85 format. -pub type Encrypted = EqlCiphertext; - // --------------------------------------------------------------------------- // Error // --------------------------------------------------------------------------- @@ -241,8 +243,8 @@ impl std::fmt::Display for JsonPathHint { #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("Configuration error: {0}")] - Config(String), + #[error("Credential error: {0}")] + Credentials(String), #[error(transparent)] ZeroKMSBuilder(#[from] ZeroKMSBuilderError), #[error(transparent)] @@ -286,12 +288,26 @@ pub enum Error { reason: JsonPathReason, hint: JsonPathHint, }, + #[error(transparent)] + Config(#[from] ConfigError), #[error("Configuration error for column '{table}.{column}': ste_vec index requires cast_as: 'json', but found cast_as: '{found_cast_as}'. Either change cast_as to 'json' or remove the ste_vec index.")] SteVecRequiresJsonCastAs { table: String, column: String, found_cast_as: String, }, + #[error("invalid eqlVersion {0}: expected 2 or 3")] + InvalidEqlVersion(u8), + #[error("Column '{column}' has no EQL v3 column type: {reason}. {hint}")] + NoV3Domain { + column: String, + reason: String, + hint: String, + }, + #[error("EQL v3 conversion failed: {0}")] + FromV2(#[from] eql_bindings::from_v2::FromV2Error), + #[error("invalid ciphertext: {0}")] + InvalidCiphertext(#[from] zerokms::DecryptError), #[error("null pointer error")] NullPointer, #[error("utf8 conversion error")] @@ -334,7 +350,7 @@ impl CredentialOpts { match (self.client_id.as_ref(), self.client_key.as_ref()) { (Some(id), Some(key)) => SecretKey::from_hex(id.clone(), key.clone()) .map(Some) - .map_err(|e| Error::Config(e.to_string())), + .map_err(|e| Error::Credentials(e.to_string())), _ => Ok(None), } } @@ -362,8 +378,11 @@ struct ClientOpts { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct NewClientOptions { - encrypt_config: EncryptConfig, + encrypt_config: CanonicalEncryptionConfig, client_opts: Option<ClientOpts>, + auth_strategy: Option<auth::AuthStrategyOpts>, + /// EQL wire version to emit: 2 (default) or 3. Validated before any I/O. + eql_version: Option<u8>, } // --------------------------------------------------------------------------- @@ -384,7 +403,6 @@ struct EncryptOptions { column: String, table: String, lock_context: Option<LockContext>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } @@ -392,7 +410,6 @@ struct EncryptOptions { #[serde(rename_all = "camelCase")] struct EncryptBulkOptions { plaintexts: Vec<PlaintextPayload>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } @@ -414,13 +431,12 @@ struct EncryptQueryOptions { plaintext: GoPlaintext, column: String, table: String, - /// The index type to use: "ste_vec", "match", "ore", "unique" + /// The index type to use: "ste_vec", "match", "ore", "ope", "unique" index_type: String, /// The query operation: "default", "ste_vec_selector", "ste_vec_term" #[serde(default = "default_query_op")] query_op: String, lock_context: Option<LockContext>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } @@ -433,7 +449,6 @@ fn default_query_op() -> String { #[serde(rename_all = "camelCase")] struct EncryptQueryBulkOptions { queries: Vec<QueryPayload>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } @@ -453,9 +468,10 @@ struct QueryPayload { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct DecryptOptions { - ciphertext: Encrypted, + /// Raw JSON payload — parsed internally so decrypt accepts BOTH the v2 and + /// v3 wire formats regardless of the client's `eqlVersion`. + ciphertext: serde_json::Value, lock_context: Option<LockContext>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } @@ -463,14 +479,14 @@ struct DecryptOptions { #[serde(rename_all = "camelCase")] struct DecryptBulkOptions { ciphertexts: Vec<BulkDecryptPayload>, - service_token: Option<ServiceToken>, unverified_context: Option<UnverifiedContext>, } #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct BulkDecryptPayload { - ciphertext: Encrypted, + /// Raw JSON payload — see [`DecryptOptions::ciphertext`]. + ciphertext: serde_json::Value, lock_context: Option<LockContext>, } @@ -567,6 +583,7 @@ fn index_type_description(index_type: &str) -> &'static str { match index_type { "ste_vec" => "JSON path and containment queries", "ore" => "range comparisons (<, >, <=, >=)", + "ope" => "range comparisons (<, >, <=, >=)", "match" => "full-text search queries", "unique" => "exact match queries", _ => "unknown query type", @@ -582,6 +599,7 @@ fn format_available_indexes(column_config: &ColumnConfig) -> String { IndexType::SteVec { .. } => "ste_vec", IndexType::Match { .. } => "match", IndexType::Ore => "ore", + IndexType::Ope => "ope", IndexType::Unique { .. } => "unique", }) .collect(); @@ -608,6 +626,7 @@ fn find_index_for_type<'a>( (IndexType::SteVec { .. }, "ste_vec") | (IndexType::Match { .. }, "match") | (IndexType::Ore, "ore") + | (IndexType::Ope, "ope") | (IndexType::Unique { .. }, "unique") ) }) @@ -636,54 +655,49 @@ fn parse_query_op(query_op: &str) -> Result<QueryOp, Error> { } /// Inferred operation mode for query encryption. -/// -/// This determines which EqlOperation to use: -/// - QueryMode: Use EqlOperation::Query (standard query encryption) -/// - StoreMode: Use EqlOperation::Store (for containment queries that need sv array) #[derive(Debug, Clone, Copy)] enum InferredQueryMode { /// Use EqlOperation::Query with the given QueryOp QueryMode(QueryOp), - /// Use EqlOperation::Store (for JSON containment queries on ste_vec) + /// Use EqlOperation::Store (for JSON containment queries on ste_vec, and for + /// v3 scalar Default queries whose operand must carry every domain term) StoreMode, } /// Convert GoPlaintext to Plaintext and infer the appropriate operation mode. /// -/// Returns both the converted Plaintext and the inferred operation mode. -/// /// Query mode has different type semantics than storage mode: /// - SteVecSelector: Always string (JSON path like "$.user.email") -> QueryMode /// - SteVecTerm: Always JSON (fragment to match with @>) -> StoreMode (produces sv array) /// - Default: For SteVec indexes, infers from plaintext type: /// - String -> QueryMode with SteVecSelector (path queries) -/// - JsonB (Object/Array) -> StoreMode (containment queries need sv array) -/// - Other indexes use column's cast_type and QueryMode with Default +/// - Json (Object/Array) -> StoreMode (containment queries need sv array) +/// - Other indexes use column's cast_type; QueryMode with Default under +/// eqlVersion 2, StoreMode under eqlVersion 3 (the v3 scalar query operand +/// must carry ALL the column domain's terms, generated exactly as storage +/// encryption generates them and hoisted by query_output). fn to_query_plaintext( go_plaintext: &GoPlaintext, query_op: QueryOp, index_type: &IndexType, column_type: cipherstash_client::schema::column::ColumnType, + eql_version: EqlVersion, ) -> Result<(Plaintext, InferredQueryMode), Error> { use cipherstash_client::schema::column::ColumnType; match query_op { QueryOp::SteVecSelector => { - // Selector queries expect a string path like "$.user.email" - // Validate the path if we have a string if let GoPlaintext::String(path) = go_plaintext { validate_json_path(path)?; } - // Force Utf8Str conversion regardless of column type - let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Utf8Str)?; + // Force Text conversion regardless of column type + let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Text)?; Ok(( plaintext, InferredQueryMode::QueryMode(QueryOp::SteVecSelector), )) } QueryOp::SteVecTerm => { - // Term queries expect a JSON fragment to match with @> - // Provide helpful errors for wrong types match go_plaintext { GoPlaintext::String(s) => { return Err(Error::InvalidQueryInput { @@ -696,7 +710,7 @@ fn to_query_plaintext( GoPlaintext::Number(n) => { return Err(Error::InvalidQueryInput { query_op: QueryOpKind::SteVecTerm, - received: ReceivedKind::Number(*n), + received: ReceivedKind::Number(n.as_f64().unwrap_or(f64::NAN)), expected: ExpectedKind::JsonObjectOrArray, hint: QueryInputHint::WrapNumberInObject, }); @@ -710,35 +724,30 @@ fn to_query_plaintext( }); } GoPlaintext::JsonB(_) => { - // This is the expected type - proceed + // Expected type - proceed } } - // Use Store mode to produce sv array for containment matching - let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::JsonB)?; + let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Json)?; Ok((plaintext, InferredQueryMode::StoreMode)) } QueryOp::Default => { - // For SteVec indexes with Default queryOp, infer from plaintext type if matches!(index_type, IndexType::SteVec { .. }) { match go_plaintext { GoPlaintext::String(path) => { - // String -> selector (path queries like "$.user.email") validate_json_path(path)?; - let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Utf8Str)?; + let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Text)?; Ok(( plaintext, InferredQueryMode::QueryMode(QueryOp::SteVecSelector), )) } GoPlaintext::JsonB(_) => { - // Object/Array -> Store mode for containment queries - // This produces sv array needed for @> operator matching - let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::JsonB)?; + let plaintext = go_plaintext.to_plaintext_with_type(ColumnType::Json)?; Ok((plaintext, InferredQueryMode::StoreMode)) } GoPlaintext::Number(n) => Err(Error::InvalidQueryInput { query_op: QueryOpKind::SteVecDefault, - received: ReceivedKind::Number(*n), + received: ReceivedKind::Number(n.as_f64().unwrap_or(f64::NAN)), expected: ExpectedKind::StringPathOrJsonObjectOrArray, hint: QueryInputHint::UsePathOrObject, }), @@ -750,23 +759,106 @@ fn to_query_plaintext( }), } } else { - // Non-SteVec indexes: use column's storage type (original behavior) let plaintext = go_plaintext.to_plaintext_with_type(column_type)?; - Ok((plaintext, InferredQueryMode::QueryMode(QueryOp::Default))) + let mode = match eql_version { + EqlVersion::V2 => InferredQueryMode::QueryMode(QueryOp::Default), + // v3 scalar operands need every term of the column's domain, + // so run Store mode and let query_output hoist them. + EqlVersion::V3 => InferredQueryMode::StoreMode, + }; + Ok((plaintext, mode)) } } } } +/// Resolve a query payload's column config and build its [`PreparedPlaintext`]. +/// +/// The single seam shared by both encrypt-query entry points, so the +/// version-dependent mode logic can never diverge. Returns the resolved +/// `&ColumnConfig` alongside the prepared plaintext — the caller needs it again +/// for [`query_output`]. +fn prepare_query_plaintext<'a>( + encrypt_config: &'a HashMap<Identifier, ColumnConfig>, + table: &str, + column: &str, + go_plaintext: &GoPlaintext, + index_type_name: &str, + query_op_name: &str, + eql_version: EqlVersion, +) -> Result<(PreparedPlaintext<'a>, &'a ColumnConfig), Error> { + let ident = Identifier::new(table.to_string(), column.to_string()); + let column_config = encrypt_config + .get(&ident) + .ok_or(Error::UnknownColumn(ident))?; + + let index = find_index_for_type(column_config, column, index_type_name)?; + let query_op = parse_query_op(query_op_name)?; + + let (plaintext, inferred_mode) = to_query_plaintext( + go_plaintext, + query_op, + &index.index_type, + column_config.cast_type, + eql_version, + )?; + + let eql_operation = match inferred_mode { + InferredQueryMode::QueryMode(qop) => EqlOperation::Query(&index.index_type, qop), + InferredQueryMode::StoreMode => EqlOperation::Store, + }; + + Ok(( + PreparedPlaintext::new( + Cow::Borrowed(column_config), + EqlIdentifier::new(table, column), + plaintext, + eql_operation, + ), + column_config, + )) +} + // --------------------------------------------------------------------------- // Core async implementations // --------------------------------------------------------------------------- -async fn new_client_impl(opts: NewClientOptions) -> Result<Client, Error> { +async fn new_client_impl( + opts: NewClientOptions, + callback: Option<GoTokenCallback>, +) -> Result<Client, Error> { + // Validate before any network I/O: a bad eqlVersion fails fast. + let eql_version = validate_eql_version(opts.eql_version)?; let client_opts = opts.client_opts.unwrap_or_default(); - let strategy = client_opts.creds.build_strategy()?; - let zerokms = ZeroKMSBuilder::new(strategy) + let auth = match opts.auth_strategy { + None => GoAuthStrategy::Auto(Box::new(client_opts.creds.build_strategy()?)), + Some(strat) => { + let cb = callback.ok_or_else(|| { + Error::Credentials("auth strategy requires a token callback".to_string()) + })?; + match strat.strategy_type { + AuthStrategyType::OidcFederation => { + let crn = client_opts.creds.workspace_crn.clone().ok_or_else(|| { + Error::Credentials( + "workspaceCrn is required for the oidcFederation auth strategy" + .to_string(), + ) + })?; + let strategy = + stack_auth::OidcFederationStrategy::builder(crn, GoOidcProvider::new(cb)) + .maybe_base_url(strat.base_url)? + .build()?; + GoAuthStrategy::Oidc(Box::new(strategy)) + } + AuthStrategyType::TokenProvider => { + GoAuthStrategy::Provided(GoProvidedTokenStrategy::new(cb)) + } + } + } + }; + + let zerokms = ZeroKMSBuilder::new(auth) .with_key_provider(client_opts.creds.build_key_provider()?) .build() .await?; @@ -774,16 +866,39 @@ async fn new_client_impl(opts: NewClientOptions) -> Result<Client, Error> { let zerokms = Arc::new(zerokms); let cipher = ScopedZeroKMS::init(zerokms.clone(), client_opts.keyset).await?; - let client = Client { + Ok(Client { cipher: Arc::new(cipher), zerokms, - encrypt_config: Arc::new(opts.encrypt_config.into_config_map()?), - }; + encrypt_config: Arc::new(build_config_map(opts.encrypt_config)?), + eql_version, + }) +} - Ok(client) +/// Turn the canonical config into the per-column map. +/// +/// `ConfigError::SteVecRequiresJson` is remapped to +/// [`Error::SteVecRequiresJsonCastAs`] so its Display keeps the +/// `ste_vec index requires cast_as` substring the Go side matches on (upstream +/// phrases it as `requires plaintext_type: json`). Every other config error +/// passes through transparently. +fn build_config_map( + config: CanonicalEncryptionConfig, +) -> Result<HashMap<Identifier, ColumnConfig>, Error> { + config.into_config_map().map_err(|e| match e { + ConfigError::SteVecRequiresJson { + table, + column, + found_plaintext_type, + } => Error::SteVecRequiresJsonCastAs { + table, + column, + found_cast_as: found_plaintext_type, + }, + other => Error::Config(other), + }) } -async fn encrypt_impl(client: &Client, opts: EncryptOptions) -> Result<Encrypted, Error> { +async fn encrypt_impl(client: &Client, opts: EncryptOptions) -> Result<EncryptedOutput, Error> { let ident = Identifier::new(opts.table.clone(), opts.column.clone()); let column_config = client @@ -806,21 +921,23 @@ async fn encrypt_impl(client: &Client, opts: EncryptOptions) -> Result<Encrypted let eql_opts = EqlEncryptOpts { keyset_id: None, lock_context: Cow::Owned(opts.lock_context.map(Into::into).unwrap_or_default()), - service_token: opts.service_token.map(Cow::Owned), unverified_context: opts.unverified_context.map(Cow::Owned), index_types: None, + decryption_policy: None, }; let mut encrypted = encrypt_eql(client.cipher.clone(), vec![prepared], &eql_opts).await?; - Ok(encrypted.remove(0)) + let eql_ciphertext = into_store_ciphertext(encrypted.remove(0))?; + + storage_output(eql_ciphertext, client.eql_version, column_config) } async fn encrypt_bulk_impl( client: &Client, opts: EncryptBulkOptions, -) -> Result<Vec<Encrypted>, Error> { - // Group payloads by lock_context for batch processing - // BTreeMap provides deterministic ordering of groups +) -> Result<Vec<EncryptedOutput>, Error> { + // Group payloads by lock_context for batch processing. + // BTreeMap provides deterministic ordering of groups. let mut groups: BTreeMap<Vec<String>, Vec<(usize, PlaintextPayload)>> = BTreeMap::new(); for (idx, payload) in opts.plaintexts.into_iter().enumerate() { @@ -832,18 +949,15 @@ async fn encrypt_bulk_impl( groups.entry(key).or_default().push((idx, payload)); } - // Pre-allocate results vector let total_count: usize = groups.values().map(|g| g.len()).sum(); - let mut results: Vec<Option<EqlCiphertext>> = (0..total_count).map(|_| None).collect(); + let mut results: Vec<Option<EncryptedOutput>> = (0..total_count).map(|_| None).collect(); - // Process each lock_context group for (lock_context_claims, payloads) in groups { let lock_context: Vec<zerokms::Context> = lock_context_claims .into_iter() .map(zerokms::Context::IdentityClaim) .collect(); - // Build PreparedPlaintext items for this group let mut prepared_plaintexts = Vec::with_capacity(payloads.len()); let mut payload_data: Vec<(usize, Identifier)> = Vec::with_capacity(payloads.len()); @@ -874,20 +988,26 @@ async fn encrypt_bulk_impl( let eql_opts = EqlEncryptOpts { keyset_id: None, lock_context: Cow::Owned(lock_context), - service_token: opts.service_token.as_ref().map(Cow::Borrowed), unverified_context: opts.unverified_context.as_ref().map(Cow::Borrowed), index_types: None, + decryption_policy: None, }; let encrypted = encrypt_eql(client.cipher.clone(), prepared_plaintexts, &eql_opts).await?; - // Place results back in original order - for (eql_ciphertext, (original_idx, _ident)) in encrypted.into_iter().zip(payload_data) { - results[original_idx] = Some(eql_ciphertext); + for (eql_output, (original_idx, ident)) in encrypted.into_iter().zip(payload_data) { + let column_config = client + .encrypt_config + .get(&ident) + .ok_or_else(|| Error::UnknownColumn(ident.clone()))?; + results[original_idx] = Some(storage_output( + into_store_ciphertext(eql_output)?, + client.eql_version, + column_config, + )?); } } - // Unwrap all results (all should be Some) results .into_iter() .enumerate() @@ -902,57 +1022,35 @@ async fn encrypt_bulk_impl( async fn encrypt_query_impl( client: &Client, opts: EncryptQueryOptions, -) -> Result<EqlCiphertext, Error> { - let ident = Identifier::new(opts.table.clone(), opts.column.clone()); - - let column_config = client - .encrypt_config - .get(&ident) - .ok_or_else(|| Error::UnknownColumn(ident.clone()))?; - - // Find the requested index type from column config - let index = find_index_for_type(column_config, &opts.column, &opts.index_type)?; - let query_op = parse_query_op(&opts.query_op)?; - - // Infer type and operation mode from plaintext - let (plaintext, inferred_mode) = to_query_plaintext( +) -> Result<QueryOutput, Error> { + let (prepared, column_config) = prepare_query_plaintext( + &client.encrypt_config, + &opts.table, + &opts.column, &opts.plaintext, - query_op, - &index.index_type, - column_config.cast_type, + &opts.index_type, + &opts.query_op, + client.eql_version, )?; - // Select the appropriate EqlOperation based on inferred mode - let eql_operation = match inferred_mode { - InferredQueryMode::QueryMode(qop) => EqlOperation::Query(&index.index_type, qop), - InferredQueryMode::StoreMode => EqlOperation::Store, - }; - - let eql_ident = EqlIdentifier::new(&opts.table, &opts.column); - let prepared = PreparedPlaintext::new( - Cow::Borrowed(column_config), - eql_ident, - plaintext, - eql_operation, - ); - let eql_opts = EqlEncryptOpts { keyset_id: None, lock_context: Cow::Owned(opts.lock_context.map(Into::into).unwrap_or_default()), - service_token: opts.service_token.map(Cow::Owned), unverified_context: opts.unverified_context.map(Cow::Owned), index_types: None, + decryption_policy: None, }; let mut encrypted = encrypt_eql(client.cipher.clone(), vec![prepared], &eql_opts).await?; - Ok(encrypted.remove(0)) + let eql_output = encrypted.remove(0); + + query_output(eql_output, client.eql_version, column_config) } async fn encrypt_query_bulk_impl( client: &Client, opts: EncryptQueryBulkOptions, -) -> Result<Vec<EqlCiphertext>, Error> { - // Group payloads by lock_context (same pattern as encrypt_bulk) +) -> Result<Vec<QueryOutput>, Error> { let mut groups: BTreeMap<Vec<String>, Vec<(usize, QueryPayload)>> = BTreeMap::new(); for (idx, payload) in opts.queries.into_iter().enumerate() { @@ -965,7 +1063,7 @@ async fn encrypt_query_bulk_impl( } let total_count: usize = groups.values().map(|g| g.len()).sum(); - let mut results: Vec<Option<EqlCiphertext>> = (0..total_count).map(|_| None).collect(); + let mut results: Vec<Option<QueryOutput>> = (0..total_count).map(|_| None).collect(); for (lock_context_claims, payloads) in groups { let lock_context: Vec<zerokms::Context> = lock_context_claims @@ -974,54 +1072,36 @@ async fn encrypt_query_bulk_impl( .collect(); let mut prepared_plaintexts = Vec::with_capacity(payloads.len()); - let mut original_indices = Vec::with_capacity(payloads.len()); - - for (original_idx, payload) in payloads { - let ident = Identifier::new(payload.table.clone(), payload.column.clone()); - let column_config = client - .encrypt_config - .get(&ident) - .ok_or_else(|| Error::UnknownColumn(ident.clone()))?; + let mut payload_data: Vec<(usize, &ColumnConfig)> = Vec::with_capacity(payloads.len()); - let index = find_index_for_type(column_config, &payload.column, &payload.index_type)?; - let query_op = parse_query_op(&payload.query_op)?; - - let (plaintext, inferred_mode) = to_query_plaintext( + for (original_idx, payload) in &payloads { + let (prepared, column_config) = prepare_query_plaintext( + &client.encrypt_config, + &payload.table, + &payload.column, &payload.plaintext, - query_op, - &index.index_type, - column_config.cast_type, + &payload.index_type, + &payload.query_op, + client.eql_version, )?; - let eql_operation = match inferred_mode { - InferredQueryMode::QueryMode(qop) => EqlOperation::Query(&index.index_type, qop), - InferredQueryMode::StoreMode => EqlOperation::Store, - }; - - let eql_ident = EqlIdentifier::new(&payload.table, &payload.column); - let prepared = PreparedPlaintext::new( - Cow::Borrowed(column_config), - eql_ident, - plaintext, - eql_operation, - ); - prepared_plaintexts.push(prepared); - original_indices.push(original_idx); + payload_data.push((*original_idx, column_config)); } let eql_opts = EqlEncryptOpts { keyset_id: None, lock_context: Cow::Owned(lock_context), - service_token: opts.service_token.as_ref().map(Cow::Borrowed), unverified_context: opts.unverified_context.as_ref().map(Cow::Borrowed), index_types: None, + decryption_policy: None, }; let encrypted = encrypt_eql(client.cipher.clone(), prepared_plaintexts, &eql_opts).await?; - for (eql_ciphertext, original_idx) in encrypted.into_iter().zip(original_indices) { - results[original_idx] = Some(eql_ciphertext); + for (eql_output, (original_idx, column_config)) in encrypted.into_iter().zip(payload_data) { + results[original_idx] = + Some(query_output(eql_output, client.eql_version, column_config)?); } } @@ -1038,16 +1118,11 @@ async fn encrypt_query_bulk_impl( async fn decrypt_impl(client: &Client, opts: DecryptOptions) -> Result<GoPlaintext, Error> { let lock_context = opts.lock_context.map(Into::into).unwrap_or_default(); - let encrypted_record = encrypted_record_from_mp_base85(opts.ciphertext, lock_context)?; + let encrypted_record = encrypted_record_from_value(opts.ciphertext, lock_context)?; let plaintext = client .zerokms - .decrypt_single( - encrypted_record, - None, - opts.service_token.map(Cow::Owned), - opts.unverified_context.as_ref(), - ) + .decrypt_single(encrypted_record, None, opts.unverified_context.as_ref()) .await .map_err(Error::from) .and_then(|bytes| Plaintext::from_slice(bytes.as_slice()).map_err(Error::from))?; @@ -1059,30 +1134,18 @@ async fn decrypt_bulk_impl( client: &Client, opts: DecryptBulkOptions, ) -> Result<Vec<GoPlaintext>, Error> { - let ciphertexts: Vec<(Encrypted, Vec<zerokms::Context>)> = opts + let encrypted_records: Vec<WithContext<'static>> = opts .ciphertexts .into_iter() .map(|payload| { let lock_context = payload.lock_context.map(Into::into).unwrap_or_default(); - (payload.ciphertext, lock_context) - }) - .collect(); - - let encrypted_records: Vec<WithContext<'static>> = ciphertexts - .into_iter() - .map(|(ciphertext, encryption_context)| { - encrypted_record_from_mp_base85(ciphertext, encryption_context) + encrypted_record_from_value(payload.ciphertext, lock_context) }) .collect::<Result<Vec<_>, Error>>()?; let decrypted = client .zerokms - .decrypt( - encrypted_records, - None, - opts.service_token.map(Cow::Owned), - opts.unverified_context.as_ref(), - ) + .decrypt(encrypted_records, None, opts.unverified_context.as_ref()) .await?; let plaintexts = decrypted @@ -1097,66 +1160,95 @@ async fn decrypt_bulk_fallible_impl( client: &Client, opts: DecryptBulkOptions, ) -> Result<Vec<DecryptResult>, Error> { - let ciphertexts: Vec<(Encrypted, Vec<zerokms::Context>)> = opts + // Decode each ciphertext independently so a single invalid payload turns + // into a per-item error rather than aborting the whole batch. + let parsed: Vec<Result<WithContext<'static>, Error>> = opts .ciphertexts .into_iter() .map(|payload| { let lock_context = payload.lock_context.map(Into::into).unwrap_or_default(); - (payload.ciphertext, lock_context) + encrypted_record_from_value(payload.ciphertext, lock_context) }) .collect(); - let encrypted_records: Vec<WithContext<'static>> = ciphertexts - .into_iter() - .map(|(ciphertext, encryption_context)| { - encrypted_record_from_mp_base85(ciphertext, encryption_context) - }) - .collect::<Result<Vec<_>, Error>>()?; + let mut results: Vec<Option<DecryptResult>> = (0..parsed.len()).map(|_| None).collect(); + let mut valid_records: Vec<WithContext<'static>> = Vec::with_capacity(parsed.len()); + let mut valid_indices: Vec<usize> = Vec::with_capacity(parsed.len()); + + for (idx, item) in parsed.into_iter().enumerate() { + match item { + Ok(record) => { + valid_records.push(record); + valid_indices.push(idx); + } + Err(e) => { + results[idx] = Some(DecryptResult::Error { + error: e.to_string(), + }); + } + } + } let decrypted: Vec<Result<Vec<u8>, RecordDecryptError>> = client .zerokms - .decrypt_fallible( - encrypted_records, - opts.service_token.map(Cow::Owned), - opts.unverified_context.map(Cow::Owned), - ) + .decrypt_fallible(valid_records, opts.unverified_context.map(Cow::Owned)) .await?; - let plaintexts: Vec<Result<GoPlaintext, Error>> = decrypted - .into_iter() - .map(|item: Result<Vec<u8>, RecordDecryptError>| { - item.map_err(Error::from).and_then(|bytes| { - Plaintext::from_slice(&bytes) - .map_err(Error::from) - .and_then(|e| GoPlaintext::try_from(e).map_err(Error::from)) - }) - }) - .collect(); + for (item, idx) in decrypted.into_iter().zip(valid_indices) { + results[idx] = Some(match item { + Ok(bytes) => match Plaintext::from_slice(&bytes) + .map_err(Error::from) + .and_then(|p| GoPlaintext::try_from(p).map_err(Error::from)) + { + Ok(data) => DecryptResult::Success { data }, + Err(e) => DecryptResult::Error { + error: e.to_string(), + }, + }, + Err(e) => DecryptResult::Error { + error: e.to_string(), + }, + }); + } - let results = plaintexts + results .into_iter() - .map(|result| match result { - Ok(data) => DecryptResult::Success { data }, - Err(err) => DecryptResult::Error { - error: err.to_string(), - }, + .enumerate() + .map(|(i, opt)| { + opt.ok_or_else(|| { + Error::InvariantViolation(format!("missing decrypt_fallible result at index {i}")) + }) }) - .collect(); - - Ok(results) + .collect::<Result<Vec<_>, _>>() } // --------------------------------------------------------------------------- // Crypto helpers // --------------------------------------------------------------------------- -fn encrypted_record_from_mp_base85( +/// Decode a v2 [`EqlCiphertext`] into the record + lock-context pair zerokms +/// decrypts. +/// +/// The SteVec root ciphertext is always `sv[0]` (mirrors upstream +/// `SteVec::into_root_ciphertext`, which is not exposed on the wire type). +/// Shared with [`eql_v3`] via `crate::encrypted_record_from_mp_base85`. +pub(crate) fn encrypted_record_from_mp_base85( encrypted: EqlCiphertext, encryption_context: Vec<zerokms::Context>, ) -> Result<WithContext<'static>, Error> { - let encrypted_record = encrypted.body.ciphertext.ok_or_else(|| { - Error::InvariantViolation("Missing ciphertext in EQL payload".to_string()) - })?; + let encrypted_record = match encrypted { + EqlCiphertext::Encrypted(payload) => payload.ciphertext, + EqlCiphertext::SteVec(payload) => { + payload + .ste_vec + .into_iter() + .next() + .ok_or_else(|| { + Error::InvariantViolation("Missing root entry in SteVec EQL payload".to_string()) + })? + .ciphertext + } + }; Ok(WithContext { record: encrypted_record, @@ -1164,12 +1256,29 @@ fn encrypted_record_from_mp_base85( }) } +/// Extract the [`EqlCiphertext`] from a Store-mode [`EqlOutput`]. +/// +/// Used by `encrypt` / `encrypt_bulk`, which always run with +/// `EqlOperation::Store` and therefore must produce storage ciphertexts. +fn into_store_ciphertext(output: EqlOutput) -> Result<EqlCiphertext, Error> { + match output { + EqlOutput::Store(ciphertext) => Ok(ciphertext), + EqlOutput::Query(_) => Err(Error::InvariantViolation( + "encrypt_eql returned a query payload for a store-mode encryption".to_string(), + )), + } +} + // --------------------------------------------------------------------------- // Exported C FFI functions // --------------------------------------------------------------------------- #[no_mangle] -pub extern "C" fn protect_new_client(options_json: *const c_char) -> CResult { +pub extern "C" fn protect_new_client( + options_json: *const c_char, + get_token: ProtectTokenFn, + token_handle: u64, +) -> CResult { let mut result = CResult::default(); let options_str = match unsafe { c_str_to_string(options_json) } { @@ -1180,10 +1289,12 @@ pub extern "C" fn protect_new_client(options_json: *const c_char) -> CResult { } }; + let callback = get_token.map(|f| GoTokenCallback::new(f, token_handle)); + let rt = get_runtime(); match rt.block_on(async { let opts: NewClientOptions = serde_json::from_str(&options_str)?; - new_client_impl(opts).await + new_client_impl(opts, callback).await }) { Ok(client) => { let client_box = Box::new(client); @@ -1199,11 +1310,21 @@ pub extern "C" fn protect_new_client(options_json: *const c_char) -> CResult { result } -#[no_mangle] -pub extern "C" fn protect_encrypt( +/// Run an operation that parses `options_json`, executes an async impl against +/// the client, and serializes the result to JSON. Centralizes the null-check, +/// UTF-8 decode, runtime dispatch, and error stringification every exported +/// operation shares. +fn run_client_op<Opts, Out, F, Fut>( client_ptr: *const Client, options_json: *const c_char, -) -> CResult { + run: F, +) -> CResult +where + Opts: for<'de> Deserialize<'de>, + Out: Serialize, + F: FnOnce(&'static Client, Opts) -> Fut, + Fut: std::future::Future<Output = Result<Out, Error>>, +{ let mut result = CResult::default(); if client_ptr.is_null() { @@ -1211,7 +1332,9 @@ pub extern "C" fn protect_encrypt( return result; } - let client = unsafe { &*client_ptr }; + // SAFETY: null-checked above; the pointer originates from `Box::into_raw` + // in `protect_new_client` and outlives this call. + let client: &Client = unsafe { &*client_ptr }; let options_str = match unsafe { c_str_to_string(options_json) } { Ok(s) => s, @@ -1222,11 +1345,13 @@ pub extern "C" fn protect_encrypt( }; let rt = get_runtime(); - match rt.block_on(async { - let opts: EncryptOptions = serde_json::from_str(&options_str)?; - encrypt_impl(client, opts).await - }) { - Ok(encrypted) => match serde_json::to_string(&encrypted) { + let outcome = rt.block_on(async { + let opts: Opts = serde_json::from_str(&options_str)?; + run(client, opts).await + }); + + match outcome { + Ok(value) => match serde_json::to_string(&value) { Ok(json) => { result.success = true; result.data = string_to_c_str(json); @@ -1244,47 +1369,25 @@ pub extern "C" fn protect_encrypt( } #[no_mangle] -pub extern "C" fn protect_encrypt_bulk( +pub extern "C" fn protect_encrypt( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: EncryptBulkOptions = serde_json::from_str(&options_str)?; - encrypt_bulk_impl(client, opts).await - }) { - Ok(encrypted_list) => match serde_json::to_string(&encrypted_list) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } + run_client_op(client_ptr, options_json, |client, opts: EncryptOptions| { + encrypt_impl(client, opts) + }) +} - result +#[no_mangle] +pub extern "C" fn protect_encrypt_bulk( + client_ptr: *const Client, + options_json: *const c_char, +) -> CResult { + run_client_op( + client_ptr, + options_json, + |client, opts: EncryptBulkOptions| encrypt_bulk_impl(client, opts), + ) } #[no_mangle] @@ -1292,43 +1395,11 @@ pub extern "C" fn protect_encrypt_query( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: EncryptQueryOptions = serde_json::from_str(&options_str)?; - encrypt_query_impl(client, opts).await - }) { - Ok(encrypted) => match serde_json::to_string(&encrypted) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } - - result + run_client_op( + client_ptr, + options_json, + |client, opts: EncryptQueryOptions| encrypt_query_impl(client, opts), + ) } #[no_mangle] @@ -1336,43 +1407,11 @@ pub extern "C" fn protect_encrypt_query_bulk( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: EncryptQueryBulkOptions = serde_json::from_str(&options_str)?; - encrypt_query_bulk_impl(client, opts).await - }) { - Ok(encrypted_list) => match serde_json::to_string(&encrypted_list) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } - - result + run_client_op( + client_ptr, + options_json, + |client, opts: EncryptQueryBulkOptions| encrypt_query_bulk_impl(client, opts), + ) } #[no_mangle] @@ -1380,43 +1419,9 @@ pub extern "C" fn protect_decrypt( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: DecryptOptions = serde_json::from_str(&options_str)?; - decrypt_impl(client, opts).await - }) { - Ok(plaintext) => match serde_json::to_string(&plaintext) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } - - result + run_client_op(client_ptr, options_json, |client, opts: DecryptOptions| { + decrypt_impl(client, opts) + }) } #[no_mangle] @@ -1424,43 +1429,11 @@ pub extern "C" fn protect_decrypt_bulk( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: DecryptBulkOptions = serde_json::from_str(&options_str)?; - decrypt_bulk_impl(client, opts).await - }) { - Ok(plaintexts) => match serde_json::to_string(&plaintexts) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } - - result + run_client_op( + client_ptr, + options_json, + |client, opts: DecryptBulkOptions| decrypt_bulk_impl(client, opts), + ) } #[no_mangle] @@ -1468,54 +1441,24 @@ pub extern "C" fn protect_decrypt_bulk_fallible( client_ptr: *const Client, options_json: *const c_char, ) -> CResult { - let mut result = CResult::default(); - - if client_ptr.is_null() { - result.error = string_to_c_str("Client pointer is null".to_string()); - return result; - } - - let client = unsafe { &*client_ptr }; - - let options_str = match unsafe { c_str_to_string(options_json) } { - Ok(s) => s, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - return result; - } - }; - - let rt = get_runtime(); - match rt.block_on(async { - let opts: DecryptBulkOptions = serde_json::from_str(&options_str)?; - decrypt_bulk_fallible_impl(client, opts).await - }) { - Ok(results_vec) => match serde_json::to_string(&results_vec) { - Ok(json) => { - result.success = true; - result.data = string_to_c_str(json); - } - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - }, - Err(e) => { - result.error = string_to_c_str(e.to_string()); - } - } - - result + run_client_op( + client_ptr, + options_json, + |client, opts: DecryptBulkOptions| decrypt_bulk_fallible_impl(client, opts), + ) } -/// Check if a JSON value is a valid EQL ciphertext. +/// Check if a JSON value is a valid EQL ciphertext (v2 or v3 storage payload). #[no_mangle] pub extern "C" fn protect_is_encrypted(value_json: *const c_char) -> bool { let value_str = match unsafe { c_str_to_string(value_json) } { Ok(s) => s, Err(_) => return false, }; - let result: Result<EqlCiphertext, _> = serde_json::from_str(&value_str); - result.is_ok() + match serde_json::from_str::<serde_json::Value>(&value_str) { + Ok(value) => is_encrypted_value(&value), + Err(_) => false, + } } #[no_mangle] @@ -1576,9 +1519,25 @@ mod tests { mod is_encrypted_tests { use super::*; + use cipherstash_client::eql::{ + EncryptedPayload, EqlCiphertext, Identifier as EqlIdentifier, SteVecEntry, + SteVecEntryTerm, SteVecPayload, EQL_SCHEMA_VERSION, + }; + use cipherstash_client::zerokms::EncryptedRecord; use serde_json::json; use std::ffi::CString; + fn dummy_encrypted_record() -> EncryptedRecord { + EncryptedRecord { + iv: Default::default(), + ciphertext: vec![1; 16], + tag: vec![2; 16], + descriptor: "users/email".to_string(), + keyset_id: None, + decryption_policy: None, + } + } + fn check_is_encrypted(value: serde_json::Value) -> bool { let json_str = serde_json::to_string(&value).unwrap(); let c_str = CString::new(json_str).unwrap(); @@ -1586,48 +1545,60 @@ mod tests { } #[test] - fn valid_eql_ciphertext_is_encrypted() { - let valid = json!({ - "i": {"t": "users", "c": "email"}, - "v": 2 + fn valid_scalar_ciphertext_is_encrypted() { + let payload = EqlCiphertext::Encrypted(EncryptedPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "email"), + ciphertext: dummy_encrypted_record(), + hmac_256: None, + bloom_filter: None, + ore_block_u64_8_256: None, + ope_cllw: None, }); - assert!(check_is_encrypted(valid)); + let value = serde_json::to_value(&payload).unwrap(); + assert_eq!(value["k"], "ct"); + assert!(check_is_encrypted(value)); } #[test] - fn valid_eql_ciphertext_with_ste_vec_is_encrypted() { - let valid = json!({ - "i": {"t": "users", "c": "profile"}, - "v": 2, - "sv": [{"s": "deadbeef"}] + fn valid_ste_vec_ciphertext_is_encrypted() { + let payload = EqlCiphertext::SteVec(SteVecPayload { + version: EQL_SCHEMA_VERSION, + identifier: EqlIdentifier::new("users", "profile"), + ste_vec: vec![SteVecEntry { + selector: "deadbeef".into(), + ciphertext: dummy_encrypted_record(), + is_array: None, + term: SteVecEntryTerm::Hmac { + hmac_256: "feedface".into(), + }, + }], }); - assert!(check_is_encrypted(valid)); + let value = serde_json::to_value(&payload).unwrap(); + assert_eq!(value["k"], "sv"); + assert!(check_is_encrypted(value)); } #[test] fn invalid_ciphertext_is_not_encrypted() { - let invalid = json!({"random": "data"}); - assert!(!check_is_encrypted(invalid)); + assert!(!check_is_encrypted(json!({"random": "data"}))); } #[test] - fn old_format_with_k_field_is_still_valid() { - let old_format = json!({ - "k": "ct", + fn missing_discriminator_is_not_encrypted() { + assert!(!check_is_encrypted(json!({ "i": {"t": "users", "c": "email"}, "v": 2 - }); - assert!(check_is_encrypted(old_format)); + }))); } #[test] - fn old_ste_vec_format_with_k_field_is_still_valid() { - let old_format = json!({ - "k": "sv", - "i": {"t": "users", "c": "profile"}, + fn unknown_discriminator_is_not_encrypted() { + assert!(!check_is_encrypted(json!({ + "k": "wat", + "i": {"t": "users", "c": "email"}, "v": 2 - }); - assert!(check_is_encrypted(old_format)); + }))); } } @@ -1652,9 +1623,7 @@ mod tests { ("b".to_string(), Some(vec!["user:1".to_string()])), ("c".to_string(), Some(vec!["user:1".to_string()])), ]; - let groups = group_by_lock_context(payloads); - assert_eq!(groups.len(), 1); assert_eq!(groups[&vec!["user:1".to_string()]].len(), 3); } @@ -1666,9 +1635,7 @@ mod tests { ("b".to_string(), Some(vec!["user:2".to_string()])), ("c".to_string(), Some(vec!["user:1".to_string()])), ]; - let groups = group_by_lock_context(payloads); - assert_eq!(groups.len(), 2); assert_eq!(groups[&vec!["user:1".to_string()]].len(), 2); assert_eq!(groups[&vec!["user:2".to_string()]].len(), 1); @@ -1681,9 +1648,7 @@ mod tests { ("b".to_string(), None), ("c".to_string(), Some(vec!["user:1".to_string()])), ]; - let groups = group_by_lock_context(payloads); - assert_eq!(groups.len(), 2); assert_eq!(groups[&vec![]].len(), 2); assert_eq!(groups[&vec!["user:1".to_string()]].len(), 1); @@ -1696,59 +1661,119 @@ mod tests { ("b".to_string(), Some(vec!["user:1".to_string()])), ("c".to_string(), Some(vec!["user:2".to_string()])), ]; - let groups = group_by_lock_context(payloads); - let user1_group = &groups[&vec!["user:1".to_string()]]; assert_eq!(user1_group[0], (1, "b".to_string())); - let user2_group = &groups[&vec!["user:2".to_string()]]; assert_eq!(user2_group[0], (0, "a".to_string())); assert_eq!(user2_group[1], (2, "c".to_string())); } } + mod config_parsing { + use super::*; + use serde_json::json; + + fn parse_config(value: serde_json::Value) -> Result<HashMap<Identifier, ColumnConfig>, Error> { + let config: CanonicalEncryptionConfig = serde_json::from_value(value).unwrap(); + build_config_map(config) + } + + #[test] + fn canonical_config_maps_columns() { + let map = parse_config(json!({ + "v": 1, + "tables": { + "users": { + "email": { "cast_as": "text", "indexes": { "unique": {} } }, + "age": { "cast_as": "small_int", "indexes": { "ore": {} } } + } + } + })) + .unwrap(); + let email = map + .get(&Identifier::new("users", "email")) + .expect("email column"); + assert_eq!( + email.cast_type, + cipherstash_client::schema::column::ColumnType::Text + ); + let age = map + .get(&Identifier::new("users", "age")) + .expect("age column"); + assert_eq!( + age.cast_type, + cipherstash_client::schema::column::ColumnType::SmallInt + ); + } + + #[test] + fn ste_vec_on_non_json_column_keeps_go_substring() { + let err = parse_config(json!({ + "v": 1, + "tables": { + "users": { + "profile": { + "cast_as": "text", + "indexes": { "ste_vec": { "prefix": "users/profile" } } + } + } + } + })) + .unwrap_err(); + let msg = err.to_string(); + // The substring Go's inferSentinel matches on for ErrSteVecRequiresJSON. + assert!( + msg.contains("ste_vec index requires cast_as"), + "error must keep the Go sentinel substring: {msg}" + ); + assert!(msg.contains("users")); + assert!(msg.contains("profile")); + } + } + mod query_op_parsing { use super::*; #[test] fn parse_query_op_default() { - let result = parse_query_op("default"); - assert!(matches!(result, Ok(QueryOp::Default))); + assert!(matches!(parse_query_op("default"), Ok(QueryOp::Default))); } #[test] fn parse_query_op_ste_vec_selector() { - let result = parse_query_op("ste_vec_selector"); - assert!(matches!(result, Ok(QueryOp::SteVecSelector))); + assert!(matches!( + parse_query_op("ste_vec_selector"), + Ok(QueryOp::SteVecSelector) + )); } #[test] fn parse_query_op_ste_vec_term() { - let result = parse_query_op("ste_vec_term"); - assert!(matches!(result, Ok(QueryOp::SteVecTerm))); + assert!(matches!( + parse_query_op("ste_vec_term"), + Ok(QueryOp::SteVecTerm) + )); } #[test] fn parse_query_op_unknown_returns_error() { - let result = parse_query_op("unknown"); - assert!(result.is_err()); - let err = result.unwrap_err(); + let err = parse_query_op("unknown").unwrap_err(); assert!(err.to_string().contains("Unknown query operation")); } } mod find_index_for_type_tests { use super::*; - use cipherstash_client::schema::column::{Index, IndexType, Tokenizer}; + use cipherstash_client::schema::column::{ColumnMode, ColumnType, Index, IndexType, Tokenizer}; fn make_column_config_with_indexes(indexes: Vec<Index>) -> ColumnConfig { ColumnConfig { name: "test_column".to_string(), - cast_type: cipherstash_client::schema::column::ColumnType::Utf8Str, + cast_type: ColumnType::Text, indexes, in_place: false, - mode: cipherstash_client::schema::column::ColumnMode::Encrypted, + mode: ColumnMode::Encrypted, } } @@ -1758,9 +1783,9 @@ mod tests { prefix: "test".to_string(), term_filters: vec![], array_index_mode: Default::default(), + mode: Default::default(), })]); let result = find_index_for_type(&config, "test_column", "ste_vec"); - assert!(result.is_ok()); assert!(matches!( result.unwrap().index_type, IndexType::SteVec { .. } @@ -1770,9 +1795,12 @@ mod tests { #[test] fn find_ore_index() { let config = make_column_config_with_indexes(vec![Index::new(IndexType::Ore)]); - let result = find_index_for_type(&config, "test_column", "ore"); - assert!(result.is_ok()); - assert!(matches!(result.unwrap().index_type, IndexType::Ore)); + assert!(matches!( + find_index_for_type(&config, "test_column", "ore") + .unwrap() + .index_type, + IndexType::Ore + )); } #[test] @@ -1780,48 +1808,22 @@ mod tests { let config = make_column_config_with_indexes(vec![Index::new(IndexType::Unique { token_filters: vec![], })]); - let result = find_index_for_type(&config, "test_column", "unique"); - assert!(result.is_ok()); assert!(matches!( - result.unwrap().index_type, + find_index_for_type(&config, "test_column", "unique") + .unwrap() + .index_type, IndexType::Unique { .. } )); } - #[test] - fn find_match_index() { - let config = make_column_config_with_indexes(vec![Index::new(IndexType::Match { - tokenizer: Tokenizer::Standard, - token_filters: vec![], - k: 3, - m: 2048, - include_original: false, - })]); - let result = find_index_for_type(&config, "test_column", "match"); - assert!(result.is_ok()); - assert!(matches!( - result.unwrap().index_type, - IndexType::Match { .. } - )); - } - #[test] fn missing_index_returns_error() { let config = make_column_config_with_indexes(vec![Index::new(IndexType::Ore)]); - let result = find_index_for_type(&config, "test_column", "ste_vec"); - assert!(result.is_err()); - let err = result.unwrap_err(); + let err = find_index_for_type(&config, "test_column", "ste_vec").unwrap_err(); assert!(err.to_string().contains("does not have")); assert!(err.to_string().contains("test_column")); } - #[test] - fn unknown_index_type_returns_error() { - let config = make_column_config_with_indexes(vec![Index::new(IndexType::Ore)]); - let result = find_index_for_type(&config, "test_column", "invalid_type"); - assert!(result.is_err()); - } - #[test] fn missing_index_error_includes_column_and_suggestions() { let config = make_column_config_with_indexes(vec![ @@ -1834,288 +1836,142 @@ mod tests { include_original: false, }), ]); - let result = find_index_for_type(&config, "email", "ste_vec"); - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("email"), - "Error should include column name: {}", - err_msg - ); - assert!( - err_msg.contains("ste_vec"), - "Error should include requested index type: {}", - err_msg - ); - assert!( - err_msg.contains("ore"), - "Error should show available ore index: {}", - err_msg - ); - assert!( - err_msg.contains("match"), - "Error should show available match index: {}", - err_msg - ); + let err_msg = find_index_for_type(&config, "email", "ste_vec") + .unwrap_err() + .to_string(); + assert!(err_msg.contains("email")); + assert!(err_msg.contains("ste_vec")); + assert!(err_msg.contains("ore")); + assert!(err_msg.contains("match")); } } mod query_inference_tests { use super::*; use cipherstash_client::encryption::Plaintext; - use cipherstash_client::schema::column::Tokenizer; - use cipherstash_client::schema::column::{ColumnType, IndexType}; + use cipherstash_client::schema::column::{ColumnType, IndexType, Tokenizer}; - #[test] - fn test_ste_vec_default_with_string_infers_selector() { - let go_plaintext = GoPlaintext::String("$.user.email".to_string()); - let index_type = IndexType::SteVec { + fn ste_vec_index() -> IndexType { + IndexType::SteVec { prefix: "test/col".to_string(), term_filters: vec![], array_index_mode: Default::default(), - }; + mode: Default::default(), + } + } + #[test] + fn ste_vec_default_with_string_infers_selector() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::String("$.user.email".to_string()), QueryOp::Default, - &index_type, - ColumnType::JsonB, + &ste_vec_index(), + ColumnType::Json, + EqlVersion::V2, ); - assert!(matches!( result, Ok(( - Plaintext::Utf8Str(Some(_)), + Plaintext::Text(Some(_)), InferredQueryMode::QueryMode(QueryOp::SteVecSelector) )) )); } #[test] - fn test_ste_vec_default_with_object_infers_store_mode() { - let go_plaintext = GoPlaintext::JsonB(serde_json::json!({"role": "admin"})); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn ste_vec_default_with_object_infers_store_mode() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::JsonB(serde_json::json!({"role": "admin"})), QueryOp::Default, - &index_type, - ColumnType::JsonB, + &ste_vec_index(), + ColumnType::Json, + EqlVersion::V2, ); - assert!(matches!( result, - Ok((Plaintext::JsonB(Some(_)), InferredQueryMode::StoreMode)) + Ok((Plaintext::Json(Some(_)), InferredQueryMode::StoreMode)) )); } #[test] - fn test_ste_vec_default_with_array_infers_store_mode() { - let go_plaintext = GoPlaintext::JsonB(serde_json::json!(["admin", "user"])); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn ste_vec_default_with_number_returns_error() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::Number(serde_json::Number::from(42)), QueryOp::Default, - &index_type, - ColumnType::JsonB, - ); - - assert!(matches!( - result, - Ok((Plaintext::JsonB(Some(_)), InferredQueryMode::StoreMode)) - )); - } - - #[test] - fn test_ste_vec_default_with_number_returns_error() { - let go_plaintext = GoPlaintext::Number(42.0); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - - let result = to_query_plaintext( - &go_plaintext, - QueryOp::Default, - &index_type, - ColumnType::JsonB, - ); - - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("Invalid query input"), - "Error message should mention invalid input: {}", - err_msg + &ste_vec_index(), + ColumnType::Json, + EqlVersion::V2, ); + assert!(result.unwrap_err().to_string().contains("Invalid query input")); } #[test] - fn test_ste_vec_default_with_boolean_returns_error() { - let go_plaintext = GoPlaintext::Boolean(true); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn non_ste_vec_default_uses_column_type_under_v2() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::String("search term".to_string()), QueryOp::Default, - &index_type, - ColumnType::JsonB, - ); - - assert!(result.is_err()); - } - - #[test] - fn test_explicit_ste_vec_selector_uses_query_mode() { - let go_plaintext = GoPlaintext::String("$.name".to_string()); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - - let result = to_query_plaintext( - &go_plaintext, - QueryOp::SteVecSelector, - &index_type, - ColumnType::JsonB, + &IndexType::Match { + tokenizer: Tokenizer::Standard, + token_filters: vec![], + k: 6, + m: 2048, + include_original: true, + }, + ColumnType::Text, + EqlVersion::V2, ); - assert!(matches!( result, Ok(( - Plaintext::Utf8Str(Some(_)), - InferredQueryMode::QueryMode(QueryOp::SteVecSelector) + Plaintext::Text(Some(_)), + InferredQueryMode::QueryMode(QueryOp::Default) )) )); } #[test] - fn test_explicit_ste_vec_term_uses_store_mode() { - let go_plaintext = GoPlaintext::JsonB(serde_json::json!({"key": "value"})); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn scalar_default_under_v3_infers_store_mode() { let result = to_query_plaintext( - &go_plaintext, - QueryOp::SteVecTerm, - &index_type, - ColumnType::JsonB, - ); - - assert!(matches!( - result, - Ok((Plaintext::JsonB(Some(_)), InferredQueryMode::StoreMode)) - )); - } - - #[test] - fn test_non_ste_vec_default_uses_column_type() { - let go_plaintext = GoPlaintext::String("search term".to_string()); - let index_type = IndexType::Match { - tokenizer: Tokenizer::Standard, - token_filters: vec![], - k: 6, - m: 2048, - include_original: true, - }; - - let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::String("hello".to_string()), QueryOp::Default, - &index_type, - ColumnType::Utf8Str, + &IndexType::Unique { + token_filters: vec![], + }, + ColumnType::Text, + EqlVersion::V3, ); - assert!(matches!( result, - Ok(( - Plaintext::Utf8Str(Some(_)), - InferredQueryMode::QueryMode(QueryOp::Default) - )) + Ok((Plaintext::Text(Some(_)), InferredQueryMode::StoreMode)) )); } #[test] - fn test_ste_vec_term_with_string_error_is_helpful() { - let go_plaintext = GoPlaintext::String("admin".to_string()); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn ste_vec_term_with_string_error_is_helpful() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::String("admin".to_string()), QueryOp::SteVecTerm, - &index_type, - ColumnType::JsonB, + &ste_vec_index(), + ColumnType::Json, + EqlVersion::V2, ); - - assert!(result.is_err()); let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("ste_vec_term"), - "Error should mention ste_vec_term: {}", - err_msg - ); - assert!( - err_msg.contains("String"), - "Error should mention received String: {}", - err_msg - ); - assert!( - err_msg.contains("ste_vec_selector") || err_msg.contains("path"), - "Error should suggest ste_vec_selector for paths: {}", - err_msg - ); + assert!(err_msg.contains("ste_vec_term")); + assert!(err_msg.contains("String")); } #[test] - fn test_invalid_json_path_error() { - let go_plaintext = GoPlaintext::String("user.email".to_string()); - let index_type = IndexType::SteVec { - prefix: "test/col".to_string(), - term_filters: vec![], - array_index_mode: Default::default(), - }; - + fn invalid_json_path_error() { let result = to_query_plaintext( - &go_plaintext, + &GoPlaintext::String("user.email".to_string()), QueryOp::SteVecSelector, - &index_type, - ColumnType::JsonB, + &ste_vec_index(), + ColumnType::Json, + EqlVersion::V2, ); - - assert!(result.is_err()); let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("user.email"), - "Error should show the invalid path: {}", - err_msg - ); - assert!( - err_msg.contains("$.user.email") || err_msg.contains("$"), - "Error should suggest correct format with $: {}", - err_msg - ); + assert!(err_msg.contains("user.email")); + assert!(err_msg.contains('$')); } } } diff --git a/examples/basic_usage.go b/examples/basic_usage.go index 65c1cc6..da497de 100644 --- a/examples/basic_usage.go +++ b/examples/basic_usage.go @@ -58,6 +58,37 @@ func main() { } defer client.Close() + // --------------------------------------------------------------- + // Per-user identity federation (optional) + // --------------------------------------------------------------- + // + // WithOIDCFederation makes every encryption and decryption identity-aware + // without threading any per-operation context through your calls. Return a + // fresh third-party OIDC access token (a JWT) from your identity provider + // (Clerk, Auth0, Supabase, ...); the client exchanges it for a CipherStash + // service token and caches it until expiry. A workspace CRN is required, + // from WithCredentials or the CS_WORKSPACE_CRN environment variable. + // + // client, err := protect.NewClient(ctx, + // protect.WithSchemas(users), + // protect.WithCredentials(crn, accessKey, clientID, clientKey), + // protect.WithOIDCFederation(func(ctx context.Context) (string, error) { + // return identityProvider.AccessToken(ctx) // your app's IdP JWT + // }), + // ) + + // --------------------------------------------------------------- + // Ciphertext format version (optional) + // --------------------------------------------------------------- + // + // The default is EncryptedFormatV2. Select V3 only for databases + // initialized with the v3 CipherStash database schema: + // + // client, err := protect.NewClient(ctx, + // protect.WithSchemas(users), + // protect.WithEncryptedFormat(protect.EncryptedFormatV3), + // ) + // --------------------------------------------------------------- // 3. Encrypt and decrypt a model (struct-based) // --------------------------------------------------------------- @@ -143,30 +174,40 @@ func main() { // --------------------------------------------------------------- // 6. Query encryption (for searching encrypted columns) // --------------------------------------------------------------- + // + // EncryptQuery returns an opaque *protect.QueryTerm. Treat it as a value to + // bind into your SQL statement — do not inspect its shape. Depending on the + // column's index configuration it may serialize as a JSON object or a bare + // JSON string. // Exact match query - queryResult, err := client.EncryptQuery(ctx, users.Column("email"), protect.Equality, "john.doe@example.com") + queryTerm, err := client.EncryptQuery(ctx, users.Column("email"), protect.Equality, "john.doe@example.com") if err != nil { log.Fatalf("Failed to encrypt query: %v", err) } - fmt.Printf("\nEncrypted equality query (unique index: %s)\n", *queryResult.UniqueIndex) + fmt.Printf("\nEncrypted equality query term: %s\n", queryTerm) // Full-text search query - searchResult, err := client.EncryptQuery(ctx, users.Column("name"), protect.FreeTextSearch, "john") + searchTerm, err := client.EncryptQuery(ctx, users.Column("name"), protect.FreeTextSearch, "john") if err != nil { log.Fatalf("Failed to encrypt search query: %v", err) } - fmt.Printf("Encrypted match query (bloom filter length: %d)\n", len(*searchResult.MatchIndex)) + fmt.Printf("Encrypted match query term (%d bytes)\n", len(searchTerm.Bytes())) // Range query - rangeResult, err := client.EncryptQuery(ctx, users.Column("age"), protect.OrderAndRange, 25) + rangeTerm, err := client.EncryptQuery(ctx, users.Column("age"), protect.OrderAndRange, 25) if err != nil { log.Fatalf("Failed to encrypt range query: %v", err) } - fmt.Printf("Encrypted range query (ORE index length: %d)\n", len(*rangeResult.OreIndex)) + fmt.Printf("Encrypted range query term (%d bytes)\n", len(rangeTerm.Bytes())) + + // Bind a query term into a parameterized SQL statement: + // + // rows, err := db.QueryContext(ctx, + // "SELECT * FROM users WHERE email = $1", queryTerm) // Bulk query encryption bulkQueries, err := client.EncryptQueryBulk(ctx, []protect.QueryItem{ @@ -177,7 +218,7 @@ func main() { log.Fatalf("Failed to bulk encrypt queries: %v", err) } - fmt.Printf("Bulk encrypted %d queries\n", len(bulkQueries)) + fmt.Printf("Bulk encrypted %d query terms\n", len(bulkQueries)) // --------------------------------------------------------------- // 7. Bulk encrypt and decrypt individual values @@ -228,28 +269,23 @@ func main() { fmt.Printf("IsEncrypted(plain string): %v\n", protect.IsEncrypted("not encrypted")) // --------------------------------------------------------------- - // 10. Identity-aware encryption (lock context) + // 10. Identity-aware encryption // --------------------------------------------------------------- - - lockContext := &protect.LockContext{ - IdentityClaim: []string{"user:12345"}, - } - - lockedEncrypted, err := client.Encrypt(ctx, users.Column("email"), "secret-data", - protect.WithLockContext(lockContext), - ) - if err != nil { - log.Fatalf("Failed to encrypt with lock context: %v", err) - } - - lockedPlaintext, err := client.Decrypt(ctx, lockedEncrypted, - protect.WithLockContext(lockContext), - ) - if err != nil { - log.Fatalf("Failed to decrypt with lock context: %v", err) - } - - fmt.Printf("Identity-aware decrypt: %v\n", lockedPlaintext) + // + // The simplest way to bind encryption to a user's identity is + // WithOIDCFederation (see the commented client setup above): every + // operation then runs under a service token derived from that user's + // identity provider session, with no per-operation configuration. + // + // A lock context additionally ties individual ciphertexts to identity + // claims. It requires the client to authenticate with an + // identity-bearing token (i.e. WithOIDCFederation) — with plain access + // key auth the platform rejects lock-context operations. + // + // lockCtx := &protect.LockContext{IdentityClaim: []string{"sub"}} + // enc, err := client.Encrypt(ctx, users.Column("email"), "secret-data", + // protect.WithLockContext(lockCtx)) + // pt, err := client.Decrypt(ctx, enc, protect.WithLockContext(lockCtx)) // --------------------------------------------------------------- // 11. Error handling with errors.Is diff --git a/pkg/protect/callback.go b/pkg/protect/callback.go new file mode 100644 index 0000000..8d99162 --- /dev/null +++ b/pkg/protect/callback.go @@ -0,0 +1,105 @@ +package protect + +/* +#include <stdint.h> +#include <stdlib.h> +*/ +import "C" + +import ( + "context" + "encoding/json" + "fmt" + "runtime/cgo" + "time" +) + +// tokenCallbackTimeout bounds a single invocation of a token provider. It +// guards against a provider that hangs, since the native layer blocks on the +// callback return. +const tokenCallbackTimeout = 30 * time.Second + +// tokenProvider holds a caller-supplied function that returns an authentication +// token. It is registered with a cgo.Handle and looked up by the exported +// callback when the native layer requests a token. +type tokenProvider struct { + getToken func(ctx context.Context) (string, error) +} + +// protectgoGetToken is the C-callable entry point invoked by the native layer +// when it needs a fresh token. The handle identifies the per-client +// tokenProvider registered in NewClient. The returned string is a NUL-terminated +// JSON envelope allocated on the C heap (via C.CString); the native layer frees +// it. A panic in the provider is recovered and reported as a failure envelope so +// it can never unwind across the FFI boundary into Rust. +// +//export protectgoGetToken +func protectgoGetToken(handle C.uint64_t) (result *C.char) { + defer func() { + if r := recover(); r != nil { + result = C.CString(providerFailureEnvelope(fmt.Sprintf("panic in token callback: %v", r))) + } + }() + + return C.CString(tokenEnvelopeForHandle(uint64(handle))) +} + +// tokenEnvelopeForHandle resolves the tokenProvider for handle, invokes it with +// a bounded context, and returns the JSON envelope to hand back to the native +// layer. +func tokenEnvelopeForHandle(handle uint64) string { + provider, ok := cgo.Handle(handle).Value().(*tokenProvider) + if !ok || provider == nil || provider.getToken == nil { + return providerFailureEnvelope("invalid token callback handle") + } + + ctx, cancel := context.WithTimeout(context.Background(), tokenCallbackTimeout) + defer cancel() + + token, err := provider.getToken(ctx) + return buildTokenEnvelope(token, err) +} + +// buildTokenEnvelope builds the JSON envelope returned to the native layer. +// On success it is {"token":"<token>"}; on error it is a PROVIDER_ERROR failure +// envelope carrying the error message. It is a pure function so the envelope +// shape can be unit-tested without any cgo call. +func buildTokenEnvelope(token string, err error) string { + if err != nil { + return providerFailureEnvelope(err.Error()) + } + b, marshalErr := json.Marshal(struct { + Token string `json:"token"` + }{Token: token}) + if marshalErr != nil { + // A plain string token can always be marshaled; this is unreachable in + // practice, but fail closed rather than return malformed JSON. + return providerFailureEnvelope("failed to encode token envelope") + } + return string(b) +} + +// providerFailureEnvelope builds a PROVIDER_ERROR failure envelope carrying msg. +func providerFailureEnvelope(msg string) string { + type failureError struct { + Message string `json:"message"` + } + type failure struct { + Type string `json:"type"` + Error failureError `json:"error"` + } + b, err := json.Marshal(struct { + Failure failure `json:"failure"` + }{ + Failure: failure{ + Type: "PROVIDER_ERROR", + Error: failureError{Message: msg}, + }, + }) + if err != nil { + // Escaping msg failed (unreachable for valid UTF-8); return a minimal + // valid envelope so the native layer still sees a failure. + return `{"failure":{"type":"PROVIDER_ERROR","error":{"message":"token callback failed"}}}` + } + return string(b) +} diff --git a/pkg/protect/errors.go b/pkg/protect/errors.go index ace327d..7b5e74d 100644 --- a/pkg/protect/errors.go +++ b/pkg/protect/errors.go @@ -32,6 +32,21 @@ var ( // ErrSteVecRequiresJSON indicates an ste_vec index requires a JSON cast type. ErrSteVecRequiresJSON = errors.New("protect: ste_vec requires json cast type") + // ErrUnsupportedFormat indicates the column's index configuration has no + // equivalent in the selected encrypted format. This typically means a + // column uses an index or cast type that the chosen ciphertext format + // (see [WithEncryptedFormat]) cannot represent. + ErrUnsupportedFormat = errors.New("protect: unsupported encrypted format for column") + + // ErrInvalidCiphertext indicates a value passed for decryption is not a + // valid ciphertext payload. + ErrInvalidCiphertext = errors.New("protect: invalid ciphertext") + + // ErrAuthStrategy indicates a client authentication strategy is + // misconfigured — for example [WithOIDCFederation] was selected without a + // workspace CRN, or a token callback is required but was not supplied. + ErrAuthStrategy = errors.New("protect: auth strategy misconfigured") + // ErrFFI indicates an unclassified error from the FFI layer. // Use errors.Is(err, ErrFFI) to check if an error originated from the // underlying encryption engine when no more specific sentinel applies. @@ -83,6 +98,13 @@ func inferSentinel(msg string) error { return ErrInvalidJSONPath case strings.Contains(msg, "ste_vec index requires cast_as"): return ErrSteVecRequiresJSON + case strings.Contains(msg, "no EQL v3 column type"): + return ErrUnsupportedFormat + case strings.Contains(msg, "invalid ciphertext"): + return ErrInvalidCiphertext + case strings.Contains(msg, "workspaceCrn is required"), + strings.Contains(msg, "auth strategy requires a token callback"): + return ErrAuthStrategy default: return ErrFFI } diff --git a/pkg/protect/libprotect_ffi_darwin_arm64.a b/pkg/protect/libprotect_ffi_darwin_arm64.a index 1814544..08b1f7c 100644 Binary files a/pkg/protect/libprotect_ffi_darwin_arm64.a and b/pkg/protect/libprotect_ffi_darwin_arm64.a differ diff --git a/pkg/protect/model.go b/pkg/protect/model.go index a41d86f..9ccf39e 100644 --- a/pkg/protect/model.go +++ b/pkg/protect/model.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "strings" + "time" "unicode" ) @@ -475,12 +476,23 @@ func setFieldValue(field reflect.Value, val any) { return } - // Direct assignment if types are compatible. + // Direct assignment if types are compatible (covers time.Time -> time.Time). if valReflect.Type().AssignableTo(fieldType) { field.Set(valReflect) return } + // time.Time destinations accept RFC 3339 or date-only strings, which is how + // the native layer returns date and timestamp columns. + if fieldType == timeType { + if s, ok := val.(string); ok { + if t, ok := parseTime(s); ok { + field.Set(reflect.ValueOf(t)) + } + } + return + } + // Handle type coercion for JSON-deserialized values. switch fieldType.Kind() { case reflect.String: @@ -502,6 +514,8 @@ func setFieldValue(field reflect.Value, val any) { case json.Number: if i, err := n.Int64(); err == nil { field.SetInt(i) + } else if f, err := n.Float64(); err == nil { + field.SetInt(int64(f)) } } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: @@ -512,6 +526,12 @@ func setFieldValue(field reflect.Value, val any) { field.SetUint(uint64(n)) case uint64: field.SetUint(n) + case json.Number: + if i, err := n.Int64(); err == nil && i >= 0 { + field.SetUint(uint64(i)) + } else if f, err := n.Float64(); err == nil && f >= 0 { + field.SetUint(uint64(f)) + } } case reflect.Float32, reflect.Float64: switch n := val.(type) { @@ -532,3 +552,15 @@ func setFieldValue(field reflect.Value, val any) { } } } + +// parseTime parses a string returned by the native layer for a date or +// timestamp column. It accepts RFC 3339 (with or without fractional seconds) +// and date-only "YYYY-MM-DD" forms. +func parseTime(s string) (time.Time, bool) { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/pkg/protect/model_test.go b/pkg/protect/model_test.go index 4c29244..cbf3df6 100644 --- a/pkg/protect/model_test.go +++ b/pkg/protect/model_test.go @@ -2,9 +2,11 @@ package protect import ( "context" + "encoding/json" "errors" "reflect" "testing" + "time" ) // --- Test structs used across multiple tests --- @@ -446,6 +448,122 @@ func TestSetFieldValue(t *testing.T) { }) } +func TestSetFieldValueJSONNumber(t *testing.T) { + t.Parallel() + + t.Run("json.Number to int64 preserves large values", func(t *testing.T) { + t.Parallel() + var i int64 + v := reflect.ValueOf(&i).Elem() + // 9007199254740993 = 2^53 + 1, which would lose precision as float64. + setFieldValue(v, json.Number("9007199254740993")) + if i != 9007199254740993 { + t.Errorf("got %d, want 9007199254740993", i) + } + }) + + t.Run("json.Number to uint", func(t *testing.T) { + t.Parallel() + var u uint64 + v := reflect.ValueOf(&u).Elem() + setFieldValue(v, json.Number("42")) + if u != 42 { + t.Errorf("got %d, want 42", u) + } + }) + + t.Run("json.Number to float", func(t *testing.T) { + t.Parallel() + var f float64 + v := reflect.ValueOf(&f).Elem() + setFieldValue(v, json.Number("3.5")) + if f != 3.5 { + t.Errorf("got %v, want 3.5", f) + } + }) +} + +func TestSetFieldValueTime(t *testing.T) { + t.Parallel() + + t.Run("RFC3339 string to time.Time", func(t *testing.T) { + t.Parallel() + var tm time.Time + v := reflect.ValueOf(&tm).Elem() + setFieldValue(v, "2021-03-04T05:06:07Z") + want := time.Date(2021, 3, 4, 5, 6, 7, 0, time.UTC) + if !tm.Equal(want) { + t.Errorf("got %v, want %v", tm, want) + } + }) + + t.Run("date-only string to time.Time", func(t *testing.T) { + t.Parallel() + var tm time.Time + v := reflect.ValueOf(&tm).Elem() + setFieldValue(v, "2021-03-04") + want := time.Date(2021, 3, 4, 0, 0, 0, 0, time.UTC) + if !tm.Equal(want) { + t.Errorf("got %v, want %v", tm, want) + } + }) + + t.Run("time.Time value assigns directly", func(t *testing.T) { + t.Parallel() + var tm time.Time + v := reflect.ValueOf(&tm).Elem() + want := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + setFieldValue(v, want) + if !tm.Equal(want) { + t.Errorf("got %v, want %v", tm, want) + } + }) + + t.Run("pointer to time.Time from string", func(t *testing.T) { + t.Parallel() + var tm *time.Time + v := reflect.ValueOf(&tm).Elem() + setFieldValue(v, "2021-03-04T05:06:07Z") + if tm == nil { + t.Fatal("got nil pointer") + } + want := time.Date(2021, 3, 4, 5, 6, 7, 0, time.UTC) + if !tm.Equal(want) { + t.Errorf("got %v, want %v", *tm, want) + } + }) +} + +func TestParseTime(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantOK bool + wantVal time.Time + }{ + {"rfc3339", "2021-03-04T05:06:07Z", true, time.Date(2021, 3, 4, 5, 6, 7, 0, time.UTC)}, + {"rfc3339 nano", "2021-03-04T05:06:07.5Z", true, time.Date(2021, 3, 4, 5, 6, 7, 500000000, time.UTC)}, + {"date only", "2021-03-04", true, time.Date(2021, 3, 4, 0, 0, 0, 0, time.UTC)}, + {"garbage", "not a time", false, time.Time{}}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, ok := parseTime(tc.input) + if ok != tc.wantOK { + t.Fatalf("ok: got %v, want %v", ok, tc.wantOK) + } + if ok && !got.Equal(tc.wantVal) { + t.Errorf("got %v, want %v", got, tc.wantVal) + } + }) + } +} + // --- toEncrypted tests --- func TestToEncrypted(t *testing.T) { diff --git a/pkg/protect/protect.go b/pkg/protect/protect.go index ac8b9a9..a3f2093 100644 --- a/pkg/protect/protect.go +++ b/pkg/protect/protect.go @@ -34,14 +34,37 @@ package protect #cgo linux,amd64,musl LDFLAGS: -lprotect_ffi_linux_x64_musl #include "protect_ffi.h" #include <stdlib.h> +#include <stdint.h> + +// protectgoGetToken is the Go token callback exported in callback.go. It is +// declared here (a declaration, not a definition) so the static bridge +// functions below can reference it as a C function pointer of type +// ProtectTokenFn. +extern char *protectgoGetToken(uint64_t handle); + +// protectNewClientWithToken calls protect_new_client wiring the exported Go +// token callback for the given cgo.Handle value. +static struct CResult protectNewClientWithToken(const char *opts, uint64_t handle) { + return protect_new_client(opts, protectgoGetToken, handle); +} + +// protectNewClientNoToken calls protect_new_client with no token callback, +// passing a NULL function pointer and a zero handle. +static struct CResult protectNewClientNoToken(const char *opts) { + return protect_new_client(opts, NULL, 0); +} */ import "C" import ( + "bytes" "context" "encoding/json" "fmt" "io" + "os" + "runtime/cgo" "sync" + "time" "unsafe" ) @@ -59,6 +82,12 @@ var _ io.Closer = (*Client)(nil) type Client struct { mu sync.RWMutex ptr unsafe.Pointer + + // tokenHandle references the per-client token provider registered with the + // native layer when an authentication strategy callback is configured. It + // is released by Close. hasToken reports whether tokenHandle is live. + tokenHandle cgo.Handle + hasToken bool } // Close releases resources held by the client. Implements [io.Closer]. @@ -71,6 +100,10 @@ func (c *Client) Close() error { } C.protect_free_client((*C.struct_Client)(c.ptr)) c.ptr = nil + if c.hasToken { + c.tokenHandle.Delete() + c.hasToken = false + } return nil } @@ -117,17 +150,37 @@ const ( // CastAs and schema config types (exported for schema building and FFI JSON) // --------------------------------------------------------------------------- -// CastAs represents the target data type for column casting in the encryption config. +// CastAs represents the target data type for column casting in the encryption +// config. Values are normalized to their canonical form when the configuration +// is sent to the native layer, so both the canonical constants and the legacy +// aliases below produce identical wire output. +// +// Canonical types: [CastAsText], [CastAsBigInt], [CastAsInt], [CastAsSmallInt], +// [CastAsFloat], [CastAsDecimal], [CastAsBoolean], [CastAsDate], +// [CastAsTimestamp], and [CastAsJSON]. type CastAs string const ( - CastAsBigInt CastAs = "bigint" - CastAsBoolean CastAs = "boolean" - CastAsDate CastAs = "date" - CastAsNumber CastAs = "number" - CastAsString CastAs = "string" - CastAsText CastAs = "text" - CastAsJSON CastAs = "json" + // Canonical cast types. + + CastAsText CastAs = "text" + CastAsBigInt CastAs = "bigint" + CastAsInt CastAs = "int" + CastAsSmallInt CastAs = "small_int" + CastAsFloat CastAs = "float" + CastAsDecimal CastAs = "decimal" + CastAsBoolean CastAs = "boolean" + CastAsDate CastAs = "date" + CastAsTimestamp CastAs = "timestamp" + CastAsJSON CastAs = "json" + + // CastAsString is a legacy alias for [CastAsText]. It is normalized to + // "text" on the wire. + CastAsString CastAs = "string" + + // CastAsNumber is a legacy alias for [CastAsFloat]. It is normalized to + // "float" on the wire. + CastAsNumber CastAs = "number" // CastAsJson is a deprecated alias for [CastAsJSON]. // @@ -135,7 +188,23 @@ const ( CastAsJson = CastAsJSON ) -// Identifier represents a table and column identifier in the EQL wire format. +// normalizeCastAs maps a public CastAs value to its canonical wire name. +// Legacy aliases are rewritten: string→text, number→float, bigint→big_int. +// All other values are already canonical and returned unchanged. +func normalizeCastAs(c CastAs) CastAs { + switch c { + case CastAsString: + return CastAsText + case CastAsNumber: + return CastAsFloat + case CastAsBigInt: + return "big_int" + default: + return c + } +} + +// Identifier represents a table and column identifier in the encrypted wire format. type Identifier struct { Table string `json:"t"` Column string `json:"c"` @@ -208,17 +277,56 @@ type LockContext struct { } // Encrypted represents an encrypted value with its metadata and indexes. -// The JSON tags match the EQL wire format and must not be changed. +// The JSON tags match the native wire format and must not be changed. +// Fields are preserved verbatim so that a ciphertext round-tripped through Go +// is byte-for-byte compatible with what the native layer emits. type Encrypted struct { Identifier Identifier `json:"i"` Version uint16 `json:"v"` + K *string `json:"k,omitempty"` Ciphertext *string `json:"c,omitempty"` OreIndex *[]string `json:"ob,omitempty"` MatchIndex *[]uint16 `json:"bf,omitempty"` UniqueIndex *string `json:"hm,omitempty"` SteVecIndex any `json:"sv,omitempty"` + Op *string `json:"op,omitempty"` } +// QueryTerm is an opaque encrypted query term produced by [Client.EncryptQuery] +// and [Client.EncryptQueryBulk]. Bind it directly into a SQL statement as the +// search value against an encrypted column. +// +// Depending on the column's index configuration, a query term may serialize as +// a JSON object or as a bare JSON string. Treat it as an opaque value: inspect +// it with [QueryTerm.Bytes] or [QueryTerm.String], and marshal it with +// encoding/json to obtain the exact payload the database expects. Do not depend +// on its internal shape. +type QueryTerm struct { + raw json.RawMessage +} + +// MarshalJSON returns the raw query-term JSON. A zero-value QueryTerm marshals +// as JSON null. +func (q QueryTerm) MarshalJSON() ([]byte, error) { + if len(q.raw) == 0 { + return []byte("null"), nil + } + return q.raw, nil +} + +// UnmarshalJSON stores the raw JSON verbatim without interpreting its shape. +func (q *QueryTerm) UnmarshalJSON(data []byte) error { + q.raw = append(q.raw[:0], data...) + return nil +} + +// Bytes returns the raw JSON encoding of the query term. The returned slice +// must not be modified. +func (q QueryTerm) Bytes() []byte { return q.raw } + +// String returns the raw JSON encoding of the query term as a string. +func (q QueryTerm) String() string { return string(q.raw) } + // PlaintextItem is a single value for bulk encryption via [Client.EncryptBulk]. type PlaintextItem struct { // Column identifies the table and column for encryption. @@ -263,11 +371,36 @@ type clientConfig struct { clientKey string keysetName string keysetID string + + // oidcGetToken, when set, selects the OIDC federation auth strategy. + oidcGetToken func(ctx context.Context) (string, error) + // tokenProviderGetToken, when set, selects the direct token provider + // auth strategy. + tokenProviderGetToken func(ctx context.Context) (string, error) + + // encryptedFormat selects the ciphertext format version. The zero value + // means "use the default" ([EncryptedFormatV2]). + encryptedFormat EncryptedFormat } // ClientOption configures the Client during construction. type ClientOption func(*clientConfig) +// EncryptedFormat selects the on-disk ciphertext format version produced by the +// client. Use it with [WithEncryptedFormat]. +type EncryptedFormat int + +const ( + // EncryptedFormatV2 is the default ciphertext format, compatible with + // databases initialized with the v2 CipherStash database schema. + EncryptedFormatV2 EncryptedFormat = 2 + + // EncryptedFormatV3 targets databases initialized with the v3 CipherStash + // database schema. Select it only when your database has been provisioned + // for the v3 schema. + EncryptedFormatV3 EncryptedFormat = 3 +) + // WithSchemas registers one or more table schemas with the client. // The schemas define which tables and columns can be encrypted. func WithSchemas(schemas ...*TableDef) ClientOption { @@ -301,13 +434,56 @@ func WithKeysetID(id string) ClientOption { } } +// WithOIDCFederation configures per-user identity federation. The provided +// getToken returns a fresh third-party OIDC access token (a JWT) from your +// application's identity provider (Clerk, Auth0, Supabase, and similar). +// +// The client exchanges that token for a short-lived CipherStash service token +// and caches it until expiry, invoking getToken again only when it must +// re-federate. This makes every encryption and decryption identity-aware at the +// client level, without threading any per-operation context through your calls. +// +// A workspace CRN is required: supply it via [WithCredentials] or the +// CS_WORKSPACE_CRN environment variable. [NewClient] returns an error wrapping +// [ErrAuthStrategy] if neither is present. +// +// WithOIDCFederation and [WithTokenProvider] are mutually exclusive. +func WithOIDCFederation(getToken func(ctx context.Context) (string, error)) ClientOption { + return func(c *clientConfig) { + c.oidcGetToken = getToken + } +} + +// WithTokenProvider supplies a CipherStash service token directly. The provided +// getToken is called on every keyservice request and must return a valid +// service token; caching is the caller's responsibility. +// +// This is an advanced option for callers that mint or broker CipherStash +// service tokens themselves. Most applications should prefer +// [WithOIDCFederation], which handles token exchange and caching. +// +// WithTokenProvider and [WithOIDCFederation] are mutually exclusive. +func WithTokenProvider(getToken func(ctx context.Context) (string, error)) ClientOption { + return func(c *clientConfig) { + c.tokenProviderGetToken = getToken + } +} + +// WithEncryptedFormat selects the ciphertext format version. The default is +// [EncryptedFormatV2]. Select [EncryptedFormatV3] only for databases that have +// been initialized with the v3 CipherStash database schema. +func WithEncryptedFormat(f EncryptedFormat) ClientOption { + return func(c *clientConfig) { + c.encryptedFormat = f + } +} + // --------------------------------------------------------------------------- // Operation options (functional options for Encrypt/Decrypt/Query calls) // --------------------------------------------------------------------------- type callOpts struct { lockContext *LockContext - serviceToken *string unverifiedContext any } @@ -319,11 +495,6 @@ func WithLockContext(lc *LockContext) Option { return func(o *callOpts) { o.lockContext = lc } } -// WithServiceToken sets an explicit service token for the operation. -func WithServiceToken(token string) Option { - return func(o *callOpts) { o.serviceToken = &token } -} - // WithAuditContext attaches unverified context for audit logging. func WithAuditContext(ctx any) Option { return func(o *callOpts) { o.unverifiedContext = ctx } @@ -341,91 +512,201 @@ func buildCallOpts(opts []Option) callOpts { // NewClient // --------------------------------------------------------------------------- +// FFI-facing option types for NewClient. Field names use camelCase to match +// the native wire contract. +type ffiKeyset struct { + Name *string `json:"name,omitempty"` + ID *string `json:"id,omitempty"` +} + +type ffiClientOpts struct { + WorkspaceCrn *string `json:"workspaceCrn,omitempty"` + AccessKey *string `json:"accessKey,omitempty"` + ClientID *string `json:"clientId,omitempty"` + ClientKey *string `json:"clientKey,omitempty"` + Keyset *ffiKeyset `json:"keyset,omitempty"` +} + +type ffiAuthStrategy struct { + Type string `json:"type"` +} + +type ffiNewClientOptions struct { + EncryptConfig EncryptConfig `json:"encryptConfig"` + ClientOpts *ffiClientOpts `json:"clientOpts,omitempty"` + AuthStrategy *ffiAuthStrategy `json:"authStrategy,omitempty"` + EqlVersion int `json:"eqlVersion"` +} + // NewClient creates a new protect client configured with the given options. // The ctx parameter is checked for cancellation before the FFI call. func NewClient(ctx context.Context, opts ...ClientOption) (*Client, error) { + const op = "NewClient" + cfg := &clientConfig{} for _, opt := range opts { opt(cfg) } - encryptConfig := buildEncryptConfigFromSchemas(cfg.schemas) - - type ffiKeyset struct { - Name *string `json:"name,omitempty"` - ID *string `json:"id,omitempty"` - } - type ffiClientOpts struct { - WorkspaceCrn *string `json:"workspaceCrn,omitempty"` - AccessKey *string `json:"accessKey,omitempty"` - ClientID *string `json:"clientId,omitempty"` - ClientKey *string `json:"clientKey,omitempty"` - Keyset *ffiKeyset `json:"keyset,omitempty"` - } - type ffiNewClientOptions struct { - EncryptConfig EncryptConfig `json:"encryptConfig"` - ClientOpts *ffiClientOpts `json:"clientOpts,omitempty"` + // Resolve the authentication strategy. WithOIDCFederation and + // WithTokenProvider are mutually exclusive. + if cfg.oidcGetToken != nil && cfg.tokenProviderGetToken != nil { + return nil, &Error{ + Op: op, + Err: ErrAuthStrategy, + Message: "protect: NewClient: WithOIDCFederation and WithTokenProvider are mutually exclusive", + } } ffiOpts := ffiNewClientOptions{ - EncryptConfig: encryptConfig, - } - - if cfg.workspaceCRN != "" || cfg.accessKey != "" || cfg.clientID != "" || cfg.clientKey != "" || cfg.keysetName != "" || cfg.keysetID != "" { - co := &ffiClientOpts{} - if cfg.workspaceCRN != "" { - co.WorkspaceCrn = &cfg.workspaceCRN + EncryptConfig: buildEncryptConfigFromSchemas(cfg.schemas), + ClientOpts: buildClientOpts(cfg), + EqlVersion: resolveEqlVersion(cfg.encryptedFormat), + } + + var getToken func(ctx context.Context) (string, error) + switch { + case cfg.oidcGetToken != nil: + // OIDC federation requires a workspace CRN, from WithCredentials or + // the CS_WORKSPACE_CRN environment variable. Validate before the FFI + // call so the caller gets a clear, native-independent error. + crn := cfg.workspaceCRN + if crn == "" { + crn = os.Getenv("CS_WORKSPACE_CRN") } - if cfg.accessKey != "" { - co.AccessKey = &cfg.accessKey - } - if cfg.clientID != "" { - co.ClientID = &cfg.clientID + if crn == "" { + return nil, &Error{ + Op: op, + Err: ErrAuthStrategy, + Message: "protect: NewClient: WithOIDCFederation requires a workspace CRN: set it via WithCredentials or the CS_WORKSPACE_CRN environment variable (workspaceCrn is required)", + } } - if cfg.clientKey != "" { - co.ClientKey = &cfg.clientKey + if ffiOpts.ClientOpts == nil { + ffiOpts.ClientOpts = &ffiClientOpts{} } - if cfg.keysetName != "" || cfg.keysetID != "" { - ks := &ffiKeyset{} - if cfg.keysetName != "" { - ks.Name = &cfg.keysetName - } - if cfg.keysetID != "" { - ks.ID = &cfg.keysetID - } - co.Keyset = ks + if ffiOpts.ClientOpts.WorkspaceCrn == nil { + ffiOpts.ClientOpts.WorkspaceCrn = &crn } - ffiOpts.ClientOpts = co + ffiOpts.AuthStrategy = &ffiAuthStrategy{Type: "oidcFederation"} + getToken = cfg.oidcGetToken + case cfg.tokenProviderGetToken != nil: + ffiOpts.AuthStrategy = &ffiAuthStrategy{Type: "tokenProvider"} + getToken = cfg.tokenProviderGetToken } if err := ctx.Err(); err != nil { - return nil, fmt.Errorf("protect: NewClient: %w", err) + return nil, fmt.Errorf("protect: %s: %w", op, err) } optionsJSON, err := json.Marshal(ffiOpts) if err != nil { - return nil, fmt.Errorf("protect: NewClient: marshaling options: %w", err) + return nil, fmt.Errorf("protect: %s: marshaling options: %w", op, err) } cOptionsJSON := C.CString(string(optionsJSON)) defer C.free(unsafe.Pointer(cOptionsJSON)) - result := C.protect_new_client(cOptionsJSON) + // When an auth strategy callback is configured, register the provider with + // a cgo.Handle and pass the exported Go callback to the native layer. + if getToken != nil { + handle := cgo.NewHandle(&tokenProvider{getToken: getToken}) + result := C.protectNewClientWithToken(cOptionsJSON, C.uint64_t(handle)) + if !result.success { + handle.Delete() + errorStr := C.GoString(result.error) + C.protect_free_string(result.error) + return nil, newError(op, errorStr) + } + return &Client{ + ptr: unsafe.Pointer(result.data), + tokenHandle: handle, + hasToken: true, + }, nil + } + + result := C.protectNewClientNoToken(cOptionsJSON) if !result.success { errorStr := C.GoString(result.error) C.protect_free_string(result.error) - return nil, newError("NewClient", errorStr) + return nil, newError(op, errorStr) } return &Client{ptr: unsafe.Pointer(result.data)}, nil } +// resolveEqlVersion maps an EncryptedFormat to its wire version, defaulting to +// EncryptedFormatV2 when unset. +func resolveEqlVersion(f EncryptedFormat) int { + if f == 0 { + return int(EncryptedFormatV2) + } + return int(f) +} + +// fillEnvCredentials populates credential fields that were not set via +// WithCredentials from the standard environment variables. The native layer +// resolves workspace and access-key auth from the environment on its own, but +// the client key pair must be supplied by the SDK: CS_CLIENT_ID and +// CS_CLIENT_KEY are only used together — a lone half of the pair is ignored. +func fillEnvCredentials(cfg *clientConfig) { + if cfg.clientID == "" && cfg.clientKey == "" { + id, key := os.Getenv("CS_CLIENT_ID"), os.Getenv("CS_CLIENT_KEY") + if id != "" && key != "" { + cfg.clientID = id + cfg.clientKey = key + } + } + if cfg.workspaceCRN == "" { + cfg.workspaceCRN = os.Getenv("CS_WORKSPACE_CRN") + } + if cfg.accessKey == "" { + cfg.accessKey = os.Getenv("CS_ACCESS_KEY") + if cfg.accessKey == "" { + cfg.accessKey = os.Getenv("CS_CLIENT_ACCESS_KEY") + } + } +} + +// buildClientOpts assembles the optional clientOpts object, or returns nil when +// no credential fields are configured. +func buildClientOpts(cfg *clientConfig) *ffiClientOpts { + fillEnvCredentials(cfg) + if cfg.workspaceCRN == "" && cfg.accessKey == "" && cfg.clientID == "" && + cfg.clientKey == "" && cfg.keysetName == "" && cfg.keysetID == "" { + return nil + } + co := &ffiClientOpts{} + if cfg.workspaceCRN != "" { + co.WorkspaceCrn = &cfg.workspaceCRN + } + if cfg.accessKey != "" { + co.AccessKey = &cfg.accessKey + } + if cfg.clientID != "" { + co.ClientID = &cfg.clientID + } + if cfg.clientKey != "" { + co.ClientKey = &cfg.clientKey + } + if cfg.keysetName != "" || cfg.keysetID != "" { + ks := &ffiKeyset{} + if cfg.keysetName != "" { + ks.Name = &cfg.keysetName + } + if cfg.keysetID != "" { + ks.ID = &cfg.keysetID + } + co.Keyset = ks + } + return co +} + func buildEncryptConfigFromSchemas(schemas []*TableDef) EncryptConfig { tbls := make(Tables, len(schemas)) for _, td := range schemas { tbl := make(Table, len(td.columns)) for colName, col := range td.columns { - tbl[colName] = col + tbl[colName] = canonicalizeColumn(col) } tbls[td.name] = tbl } @@ -435,6 +716,30 @@ func buildEncryptConfigFromSchemas(schemas []*TableDef) EncryptConfig { } } +// canonicalizeColumn returns a copy of col ready for the wire: cast_as is +// normalized to its canonical name, and an unset ste_vec array_index_mode is +// defaulted to "none" (the native library default differs). The input column +// stored in the schema is left unmodified. +func canonicalizeColumn(col Column) Column { + out := col + if col.CastAs != nil { + norm := normalizeCastAs(*col.CastAs) + out.CastAs = &norm + } + if col.Indexes != nil && col.Indexes.SteVecIndex != nil && + col.Indexes.SteVecIndex.ArrayIndexMode == nil { + // Copy the indexes and ste_vec opts so we can inject the default + // without mutating the stored schema. + idx := *col.Indexes + sv := *col.Indexes.SteVecIndex + none := "none" + sv.ArrayIndexMode = &none + idx.SteVecIndex = &sv + out.Indexes = &idx + } + return out +} + // --------------------------------------------------------------------------- // Encrypt // --------------------------------------------------------------------------- @@ -463,16 +768,14 @@ func (c *Client) Encrypt(ctx context.Context, col ColumnRef, plaintext any, opts Column string `json:"column"` Table string `json:"table"` LockContext *LockContext `json:"lockContext,omitempty"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } ffiOpts := ffiEncryptOptions{ - Plaintext: plaintext, + Plaintext: normalizePlaintext(plaintext), Column: col.column, Table: col.table, LockContext: co.lockContext, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -528,14 +831,12 @@ func (c *Client) Decrypt(ctx context.Context, encrypted *Encrypted, opts ...Opti type ffiDecryptOptions struct { Ciphertext *Encrypted `json:"ciphertext"` LockContext *LockContext `json:"lockContext,omitempty"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } ffiOpts := ffiDecryptOptions{ Ciphertext: encrypted, LockContext: co.lockContext, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -558,7 +859,7 @@ func (c *Client) Decrypt(ctx context.Context, encrypted *Encrypted, opts ...Opti C.protect_free_string(result.data) var plaintext any - if err := json.Unmarshal([]byte(plaintextJSON), &plaintext); err != nil { + if err := decodeFFIJSON([]byte(plaintextJSON), &plaintext); err != nil { return nil, fmt.Errorf("protect: %s: unmarshaling result: %w", op, err) } @@ -595,7 +896,6 @@ func (c *Client) EncryptBulk(ctx context.Context, items []PlaintextItem, opts .. } type ffiBulkOptions struct { Plaintexts []ffiPlaintextPayload `json:"plaintexts"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } @@ -606,7 +906,7 @@ func (c *Client) EncryptBulk(ctx context.Context, items []PlaintextItem, opts .. lc = co.lockContext } payloads[i] = ffiPlaintextPayload{ - Plaintext: item.Plaintext, + Plaintext: normalizePlaintext(item.Plaintext), Column: item.Column.column, Table: item.Column.table, LockContext: lc, @@ -615,7 +915,6 @@ func (c *Client) EncryptBulk(ctx context.Context, items []PlaintextItem, opts .. ffiOpts := ffiBulkOptions{ Plaintexts: payloads, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -672,7 +971,6 @@ func (c *Client) DecryptBulk(ctx context.Context, items []*Encrypted, opts ...Op } type ffiBulkDecryptOptions struct { Ciphertexts []ffiBulkDecryptPayload `json:"ciphertexts"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } @@ -686,7 +984,6 @@ func (c *Client) DecryptBulk(ctx context.Context, items []*Encrypted, opts ...Op ffiOpts := ffiBulkDecryptOptions{ Ciphertexts: payloads, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -709,7 +1006,7 @@ func (c *Client) DecryptBulk(ctx context.Context, items []*Encrypted, opts ...Op C.protect_free_string(result.data) var plaintexts []any - if err := json.Unmarshal([]byte(plaintextJSON), &plaintexts); err != nil { + if err := decodeFFIJSON([]byte(plaintextJSON), &plaintexts); err != nil { return nil, fmt.Errorf("protect: %s: unmarshaling result: %w", op, err) } @@ -746,7 +1043,6 @@ func (c *Client) DecryptBulkFallible(ctx context.Context, items []*Encrypted, op } type ffiBulkDecryptOptions struct { Ciphertexts []ffiBulkDecryptPayload `json:"ciphertexts"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } @@ -760,7 +1056,6 @@ func (c *Client) DecryptBulkFallible(ctx context.Context, items []*Encrypted, op ffiOpts := ffiBulkDecryptOptions{ Ciphertexts: payloads, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -788,7 +1083,7 @@ func (c *Client) DecryptBulkFallible(ctx context.Context, items []*Encrypted, op } var ffiResults []ffiDecryptResult - if err := json.Unmarshal([]byte(resultsJSON), &ffiResults); err != nil { + if err := decodeFFIJSON([]byte(resultsJSON), &ffiResults); err != nil { return nil, fmt.Errorf("protect: %s: unmarshaling result: %w", op, err) } @@ -808,13 +1103,14 @@ func (c *Client) DecryptBulkFallible(ctx context.Context, items []*Encrypted, op // EncryptQuery // --------------------------------------------------------------------------- -// EncryptQuery encrypts a value for searching against an encrypted column. +// EncryptQuery encrypts a value for searching against an encrypted column and +// returns an opaque [QueryTerm] to bind into a SQL statement. // -// The queryType determines which index is used for the search. For example, -// [Equality] produces an HMAC for exact-match, [FreeTextSearch] produces a -// bloom filter for full-text search, and [OrderAndRange] produces an ORE -// ciphertext for range comparisons. -func (c *Client) EncryptQuery(ctx context.Context, col ColumnRef, queryType QueryType, plaintext any, opts ...Option) (*Encrypted, error) { +// The queryType determines which index is used for the search: [Equality] for +// exact-match, [FreeTextSearch] for full-text search, [OrderAndRange] for range +// and ordering comparisons, and the JSON query types for path and containment +// queries. The returned term should be treated as opaque; see [QueryTerm]. +func (c *Client) EncryptQuery(ctx context.Context, col ColumnRef, queryType QueryType, plaintext any, opts ...Option) (*QueryTerm, error) { const op = "EncryptQuery" ptr, unlock, err := c.acquirePtr(op) @@ -838,18 +1134,16 @@ func (c *Client) EncryptQuery(ctx context.Context, col ColumnRef, queryType Quer IndexType string `json:"indexType"` QueryOp string `json:"queryOp,omitempty"` LockContext *LockContext `json:"lockContext,omitempty"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } ffiOpts := ffiEncryptQueryOptions{ - Plaintext: plaintext, + Plaintext: normalizePlaintext(plaintext), Column: col.column, Table: col.table, IndexType: indexType, QueryOp: queryOp, LockContext: co.lockContext, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -868,23 +1162,19 @@ func (c *Client) EncryptQuery(ctx context.Context, col ColumnRef, queryType Quer return nil, newError(op, errorStr) } - encryptedJSON := C.GoString(result.data) + termJSON := C.GoString(result.data) C.protect_free_string(result.data) - var encrypted Encrypted - if err := json.Unmarshal([]byte(encryptedJSON), &encrypted); err != nil { - return nil, fmt.Errorf("protect: %s: unmarshaling result: %w", op, err) - } - - return &encrypted, nil + return &QueryTerm{raw: json.RawMessage(termJSON)}, nil } // --------------------------------------------------------------------------- // EncryptQueryBulk // --------------------------------------------------------------------------- -// EncryptQueryBulk encrypts multiple query values in a single operation. -func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts ...Option) ([]Encrypted, error) { +// EncryptQueryBulk encrypts multiple query values in a single operation. Each +// result is an opaque [QueryTerm] positioned to match its input query. +func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts ...Option) ([]*QueryTerm, error) { const op = "EncryptQueryBulk" ptr, unlock, err := c.acquirePtr(op) @@ -909,7 +1199,6 @@ func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts } type ffiBulkQueryOptions struct { Queries []ffiQueryPayload `json:"queries"` - ServiceToken *string `json:"serviceToken,omitempty"` UnverifiedContext any `json:"unverifiedContext,omitempty"` } @@ -921,7 +1210,7 @@ func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts lc = co.lockContext } payloads[i] = ffiQueryPayload{ - Plaintext: q.Plaintext, + Plaintext: normalizePlaintext(q.Plaintext), Column: q.Column.column, Table: q.Column.table, IndexType: indexType, @@ -932,7 +1221,6 @@ func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts ffiOpts := ffiBulkQueryOptions{ Queries: payloads, - ServiceToken: co.serviceToken, UnverifiedContext: co.unverifiedContext, } @@ -951,22 +1239,26 @@ func (c *Client) EncryptQueryBulk(ctx context.Context, queries []QueryItem, opts return nil, newError(op, errorStr) } - encryptedJSON := C.GoString(result.data) + termsJSON := C.GoString(result.data) C.protect_free_string(result.data) - var encrypted []Encrypted - if err := json.Unmarshal([]byte(encryptedJSON), &encrypted); err != nil { + var raw []json.RawMessage + if err := json.Unmarshal([]byte(termsJSON), &raw); err != nil { return nil, fmt.Errorf("protect: %s: unmarshaling result: %w", op, err) } - return encrypted, nil + terms := make([]*QueryTerm, len(raw)) + for i := range raw { + terms[i] = &QueryTerm{raw: raw[i]} + } + return terms, nil } // --------------------------------------------------------------------------- // IsEncrypted // --------------------------------------------------------------------------- -// IsEncrypted checks whether a value is a valid EQL encrypted payload. +// IsEncrypted reports whether a value is a valid encrypted payload. // This is a standalone function that does not require a [Client]. // // Note: this function makes a CGO call to validate the payload structure. @@ -986,6 +1278,32 @@ func IsEncrypted(value any) bool { // Internal helpers // --------------------------------------------------------------------------- +// normalizePlaintext converts Go values that need explicit wire formatting +// before encryption. time.Time values are formatted as RFC 3339 strings so the +// native layer can parse them for date and timestamp columns. A nil *time.Time +// becomes a JSON null. All other values pass through unchanged. +func normalizePlaintext(v any) any { + switch t := v.(type) { + case time.Time: + return t.Format(time.RFC3339Nano) + case *time.Time: + if t == nil { + return nil + } + return t.Format(time.RFC3339Nano) + default: + return v + } +} + +// decodeFFIJSON decodes an FFI JSON response into v using json.Number for +// numeric values, so integers beyond 2^53 survive without precision loss. +func decodeFFIJSON(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + return dec.Decode(v) +} + // resolveQueryType maps a public QueryType to the FFI's indexType and queryOp // string values. For standard index types (unique, match, ore), queryOp is // "default". For ste_vec variants, the indexType is "ste_vec" and queryOp diff --git a/pkg/protect/protect_ffi.h b/pkg/protect/protect_ffi.h index e6e65a5..c9d3b28 100644 --- a/pkg/protect/protect_ffi.h +++ b/pkg/protect/protect_ffi.h @@ -23,7 +23,19 @@ typedef struct CResult { const char *error; } CResult; - struct CResult protect_new_client(const char *options_json) ; +/** + * The C `getToken` callback: `char *(*)(uint64_t handle)`. + * + * Go returns a malloc'd (C heap, via `C.CString`) NUL-terminated JSON string, + * or NULL to signal "provider failed with no detail". + */ +typedef char *(*ProtectTokenFn)(uint64_t handle); + + +struct CResult protect_new_client(const char *options_json, + ProtectTokenFn get_token, + uint64_t token_handle) +; struct CResult protect_encrypt(const struct Client *client_ptr, const char *options_json) ; @@ -46,7 +58,7 @@ struct CResult protect_decrypt_bulk_fallible(const struct Client *client_ptr, ; /** - * Check if a JSON value is a valid EQL ciphertext. + * Check if a JSON value is a valid EQL ciphertext (v2 or v3 storage payload). */ bool protect_is_encrypted(const char *value_json) ; diff --git a/pkg/protect/protect_test.go b/pkg/protect/protect_test.go index dff2df8..91645a1 100644 --- a/pkg/protect/protect_test.go +++ b/pkg/protect/protect_test.go @@ -1,6 +1,7 @@ package protect import ( + "context" "encoding/json" "errors" "testing" @@ -55,13 +56,18 @@ func TestCastAsConstants(t *testing.T) { constant CastAs expected string }{ + {"Text", CastAsText, "text"}, {"BigInt", CastAsBigInt, "bigint"}, + {"Int", CastAsInt, "int"}, + {"SmallInt", CastAsSmallInt, "small_int"}, + {"Float", CastAsFloat, "float"}, + {"Decimal", CastAsDecimal, "decimal"}, {"Boolean", CastAsBoolean, "boolean"}, {"Date", CastAsDate, "date"}, - {"Number", CastAsNumber, "number"}, - {"String", CastAsString, "string"}, - {"Text", CastAsText, "text"}, + {"Timestamp", CastAsTimestamp, "timestamp"}, {"JSON", CastAsJSON, "json"}, + {"String (legacy alias)", CastAsString, "string"}, + {"Number (legacy alias)", CastAsNumber, "number"}, } for _, tc := range tests { @@ -75,6 +81,39 @@ func TestCastAsConstants(t *testing.T) { } } +func TestNormalizeCastAs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input CastAs + want CastAs + }{ + {"string alias to text", CastAsString, CastAsText}, + {"number alias to float", CastAsNumber, CastAsFloat}, + {"bigint to big_int", CastAsBigInt, "big_int"}, + {"text unchanged", CastAsText, CastAsText}, + {"int unchanged", CastAsInt, CastAsInt}, + {"small_int unchanged", CastAsSmallInt, CastAsSmallInt}, + {"float unchanged", CastAsFloat, CastAsFloat}, + {"decimal unchanged", CastAsDecimal, CastAsDecimal}, + {"boolean unchanged", CastAsBoolean, CastAsBoolean}, + {"date unchanged", CastAsDate, CastAsDate}, + {"timestamp unchanged", CastAsTimestamp, CastAsTimestamp}, + {"json unchanged", CastAsJSON, CastAsJSON}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := normalizeCastAs(tc.input); got != tc.want { + t.Errorf("normalizeCastAs(%q) = %q, want %q", tc.input, got, tc.want) + } + }) + } +} + // --------------------------------------------------------------------------- // QueryType tests // --------------------------------------------------------------------------- @@ -267,16 +306,6 @@ func TestWithLockContext(t *testing.T) { } } -func TestWithServiceToken(t *testing.T) { - t.Parallel() - - co := buildCallOpts([]Option{WithServiceToken("tok-abc")}) - - if co.serviceToken == nil || *co.serviceToken != "tok-abc" { - t.Errorf("serviceToken: got %v, want %q", co.serviceToken, "tok-abc") - } -} - func TestWithAuditContext(t *testing.T) { t.Parallel() @@ -303,14 +332,116 @@ func TestBuildCallOptsEmpty(t *testing.T) { if co.lockContext != nil { t.Error("lockContext: expected nil") } - if co.serviceToken != nil { - t.Error("serviceToken: expected nil") - } if co.unverifiedContext != nil { t.Error("unverifiedContext: expected nil") } } +// --------------------------------------------------------------------------- +// Auth strategy and format option tests +// --------------------------------------------------------------------------- + +func TestWithOIDCFederation(t *testing.T) { + t.Parallel() + + getToken := func(context.Context) (string, error) { return "jwt", nil } + cfg := &clientConfig{} + WithOIDCFederation(getToken)(cfg) + + if cfg.oidcGetToken == nil { + t.Fatal("oidcGetToken: expected non-nil") + } + if cfg.tokenProviderGetToken != nil { + t.Error("tokenProviderGetToken: expected nil") + } + tok, err := cfg.oidcGetToken(context.Background()) + if err != nil || tok != "jwt" { + t.Errorf("oidcGetToken() = (%q, %v), want (\"jwt\", nil)", tok, err) + } +} + +func TestWithTokenProvider(t *testing.T) { + t.Parallel() + + getToken := func(context.Context) (string, error) { return "cts-token", nil } + cfg := &clientConfig{} + WithTokenProvider(getToken)(cfg) + + if cfg.tokenProviderGetToken == nil { + t.Fatal("tokenProviderGetToken: expected non-nil") + } + if cfg.oidcGetToken != nil { + t.Error("oidcGetToken: expected nil") + } +} + +func TestWithEncryptedFormat(t *testing.T) { + t.Parallel() + + cfg := &clientConfig{} + WithEncryptedFormat(EncryptedFormatV3)(cfg) + if cfg.encryptedFormat != EncryptedFormatV3 { + t.Errorf("encryptedFormat: got %d, want %d", cfg.encryptedFormat, EncryptedFormatV3) + } +} + +func TestResolveEqlVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input EncryptedFormat + want int + }{ + {"unset defaults to v2", EncryptedFormat(0), 2}, + {"explicit v2", EncryptedFormatV2, 2}, + {"explicit v3", EncryptedFormatV3, 3}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := resolveEqlVersion(tc.input); got != tc.want { + t.Errorf("resolveEqlVersion(%d) = %d, want %d", tc.input, got, tc.want) + } + }) + } +} + +// NewClient must reject WithOIDCFederation without a workspace CRN before it +// makes any FFI call. +func TestNewClientOIDCWithoutCRN(t *testing.T) { + // Not parallel: mutates the CS_WORKSPACE_CRN environment variable. + t.Setenv("CS_WORKSPACE_CRN", "") + + _, err := NewClient(context.Background(), + WithOIDCFederation(func(context.Context) (string, error) { return "jwt", nil }), + ) + if err == nil { + t.Fatal("expected error for OIDC federation without a workspace CRN") + } + if !errors.Is(err, ErrAuthStrategy) { + t.Errorf("expected ErrAuthStrategy, got: %v", err) + } +} + +// NewClient must reject configuring both auth strategies at once before any FFI +// call. +func TestNewClientMutuallyExclusiveAuthStrategies(t *testing.T) { + t.Parallel() + + _, err := NewClient(context.Background(), + WithOIDCFederation(func(context.Context) (string, error) { return "jwt", nil }), + WithTokenProvider(func(context.Context) (string, error) { return "cts", nil }), + ) + if err == nil { + t.Fatal("expected error for mutually exclusive auth strategies") + } + if !errors.Is(err, ErrAuthStrategy) { + t.Errorf("expected ErrAuthStrategy, got: %v", err) + } +} + // --------------------------------------------------------------------------- // EncryptConfig building tests // --------------------------------------------------------------------------- @@ -458,6 +589,200 @@ func TestEncryptedJSONRoundTrip(t *testing.T) { } } +func TestEncryptedKindAndOpRoundTrip(t *testing.T) { + t.Parallel() + + k := "ct" + op := "eq" + ct := "cipher" + original := Encrypted{ + Identifier: Identifier{Table: "t", Column: "c"}, + Version: 2, + K: &k, + Ciphertext: &ct, + Op: &op, + } + + data, err := json.Marshal(original) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + // The k and op fields must appear in the wire JSON. + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("Unmarshal to map failed: %v", err) + } + if raw["k"] != "ct" { + t.Errorf("k: got %v, want %q", raw["k"], "ct") + } + if raw["op"] != "eq" { + t.Errorf("op: got %v, want %q", raw["op"], "eq") + } + + var decoded Encrypted + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if decoded.K == nil || *decoded.K != "ct" { + t.Errorf("K: got %v, want %q", decoded.K, "ct") + } + if decoded.Op == nil || *decoded.Op != "eq" { + t.Errorf("Op: got %v, want %q", decoded.Op, "eq") + } +} + +func TestEncryptedOmitsKindAndOpWhenNil(t *testing.T) { + t.Parallel() + + ct := "cipher" + enc := Encrypted{ + Identifier: Identifier{Table: "t", Column: "c"}, + Version: 2, + Ciphertext: &ct, + } + data, err := json.Marshal(enc) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if _, ok := raw["k"]; ok { + t.Error("k: should be omitted when nil") + } + if _, ok := raw["op"]; ok { + t.Error("op: should be omitted when nil") + } +} + +// --------------------------------------------------------------------------- +// QueryTerm tests +// --------------------------------------------------------------------------- + +func TestQueryTermObject(t *testing.T) { + t.Parallel() + + var qt QueryTerm + if err := json.Unmarshal([]byte(`{"hm":"abc","i":{"t":"users","c":"email"}}`), &qt); err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + + if got := qt.String(); got != `{"hm":"abc","i":{"t":"users","c":"email"}}` { + t.Errorf("String(): got %q", got) + } + + // Marshaling must reproduce the raw JSON verbatim. + out, err := json.Marshal(qt) + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + if string(out) != `{"hm":"abc","i":{"t":"users","c":"email"}}` { + t.Errorf("MarshalJSON: got %s", out) + } +} + +func TestQueryTermBareString(t *testing.T) { + t.Parallel() + + // A query term may be a bare JSON string, e.g. an ste_vec selector. + var qt QueryTerm + if err := json.Unmarshal([]byte(`"c2VsZWN0b3I="`), &qt); err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + if got := qt.String(); got != `"c2VsZWN0b3I="` { + t.Errorf("String(): got %q", got) + } + if got := string(qt.Bytes()); got != `"c2VsZWN0b3I="` { + t.Errorf("Bytes(): got %q", got) + } +} + +func TestQueryTermZeroValueMarshalsNull(t *testing.T) { + t.Parallel() + + var qt QueryTerm + out, err := json.Marshal(qt) + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + if string(out) != "null" { + t.Errorf("MarshalJSON: got %s, want null", out) + } +} + +func TestQueryTermInStruct(t *testing.T) { + t.Parallel() + + // A QueryTerm embedded in a larger payload round-trips its raw value. + type payload struct { + Term *QueryTerm `json:"term"` + } + var p payload + if err := json.Unmarshal([]byte(`{"term":{"ob":["x"]}}`), &p); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if p.Term == nil { + t.Fatal("Term: expected non-nil") + } + out, err := json.Marshal(p) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + if string(out) != `{"term":{"ob":["x"]}}` { + t.Errorf("round-trip: got %s", out) + } +} + +// --------------------------------------------------------------------------- +// Token callback envelope tests +// --------------------------------------------------------------------------- + +func TestBuildTokenEnvelopeSuccess(t *testing.T) { + t.Parallel() + + got := buildTokenEnvelope("my-token", nil) + if got != `{"token":"my-token"}` { + t.Errorf("buildTokenEnvelope: got %s", got) + } +} + +func TestBuildTokenEnvelopeFailure(t *testing.T) { + t.Parallel() + + got := buildTokenEnvelope("", errors.New("network down")) + + var env struct { + Failure struct { + Type string `json:"type"` + Error struct { + Message string `json:"message"` + } `json:"error"` + } `json:"failure"` + } + if err := json.Unmarshal([]byte(got), &env); err != nil { + t.Fatalf("failure envelope is not valid JSON: %v (%s)", err, got) + } + if env.Failure.Type != "PROVIDER_ERROR" { + t.Errorf("failure type: got %q, want %q", env.Failure.Type, "PROVIDER_ERROR") + } + if env.Failure.Error.Message != "network down" { + t.Errorf("failure message: got %q, want %q", env.Failure.Error.Message, "network down") + } +} + +func TestProviderFailureEnvelopeEscapesMessage(t *testing.T) { + t.Parallel() + + // A message containing quotes must still yield valid JSON. + got := providerFailureEnvelope(`he said "boom"`) + var env map[string]any + if err := json.Unmarshal([]byte(got), &env); err != nil { + t.Fatalf("envelope is not valid JSON: %v (%s)", err, got) + } +} + // --------------------------------------------------------------------------- // DecryptResult tests // --------------------------------------------------------------------------- @@ -551,6 +876,26 @@ func TestErrorSentinels(t *testing.T) { "ste_vec index requires cast_as to be json", ErrSteVecRequiresJSON, }, + { + "unsupported format for column", + "column \"email\": no EQL v3 column type for this configuration", + ErrUnsupportedFormat, + }, + { + "invalid ciphertext", + "invalid ciphertext: could not parse payload", + ErrInvalidCiphertext, + }, + { + "auth strategy missing callback", + "auth strategy requires a token callback", + ErrAuthStrategy, + }, + { + "auth strategy missing workspace crn", + "workspaceCrn is required for oidcFederation", + ErrAuthStrategy, + }, } for _, tc := range tests { diff --git a/pkg/protect/schema.go b/pkg/protect/schema.go index 41d8ce2..252cd5e 100644 --- a/pkg/protect/schema.go +++ b/pkg/protect/schema.go @@ -5,6 +5,7 @@ import ( "reflect" "strconv" "strings" + "time" ) // TableDef holds a parsed table schema. It is the primary reference for @@ -410,21 +411,31 @@ func parseSteVecOpts(params string) *SteVecIndexOpts { return opts } -// inferCastAs determines the CastAs value from a Go reflect.Type. +// timeType is the reflect.Type of time.Time, used to infer the timestamp cast. +var timeType = reflect.TypeOf(time.Time{}) + +// inferCastAs determines the CastAs value from a Go reflect.Type. The result is +// a public CastAs constant; it is normalized to its canonical wire name when the +// encryption config is built. func inferCastAs(fieldType reflect.Type) CastAs { // Dereference pointers. for fieldType.Kind() == reflect.Ptr { fieldType = fieldType.Elem() } + // time.Time maps to a timestamp column regardless of its struct kind. + if fieldType == timeType { + return CastAsTimestamp + } + switch fieldType.Kind() { case reflect.String: - return CastAsString + return CastAsText case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return CastAsNumber + return CastAsBigInt case reflect.Float32, reflect.Float64: - return CastAsNumber + return CastAsFloat case reflect.Bool: return CastAsBoolean default: diff --git a/pkg/protect/schema_test.go b/pkg/protect/schema_test.go index 0484e3b..3e38131 100644 --- a/pkg/protect/schema_test.go +++ b/pkg/protect/schema_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "reflect" "testing" + "time" ) // --- Tag parsing tests --- @@ -117,8 +118,8 @@ func TestParseColumnSimple(t *testing.T) { t.Parallel() col := parseColumn(nil, reflect.TypeOf("")) - if col.CastAs == nil || *col.CastAs != CastAsString { - t.Errorf("cast_as: got %v, want %q", col.CastAs, CastAsString) + if col.CastAs == nil || *col.CastAs != CastAsText { + t.Errorf("cast_as: got %v, want %q", col.CastAs, CastAsText) } if col.Indexes != nil { t.Errorf("indexes: expected nil, got %+v", col.Indexes) @@ -284,25 +285,27 @@ func TestInferCastAs(t *testing.T) { typ reflect.Type wantCast CastAs }{ - {"string", reflect.TypeOf(""), CastAsString}, - {"int", reflect.TypeOf(0), CastAsNumber}, - {"int8", reflect.TypeOf(int8(0)), CastAsNumber}, - {"int16", reflect.TypeOf(int16(0)), CastAsNumber}, - {"int32", reflect.TypeOf(int32(0)), CastAsNumber}, - {"int64", reflect.TypeOf(int64(0)), CastAsNumber}, - {"uint", reflect.TypeOf(uint(0)), CastAsNumber}, - {"uint8", reflect.TypeOf(uint8(0)), CastAsNumber}, - {"uint16", reflect.TypeOf(uint16(0)), CastAsNumber}, - {"uint32", reflect.TypeOf(uint32(0)), CastAsNumber}, - {"uint64", reflect.TypeOf(uint64(0)), CastAsNumber}, - {"float32", reflect.TypeOf(float32(0)), CastAsNumber}, - {"float64", reflect.TypeOf(float64(0)), CastAsNumber}, + {"string", reflect.TypeOf(""), CastAsText}, + {"int", reflect.TypeOf(0), CastAsBigInt}, + {"int8", reflect.TypeOf(int8(0)), CastAsBigInt}, + {"int16", reflect.TypeOf(int16(0)), CastAsBigInt}, + {"int32", reflect.TypeOf(int32(0)), CastAsBigInt}, + {"int64", reflect.TypeOf(int64(0)), CastAsBigInt}, + {"uint", reflect.TypeOf(uint(0)), CastAsBigInt}, + {"uint8", reflect.TypeOf(uint8(0)), CastAsBigInt}, + {"uint16", reflect.TypeOf(uint16(0)), CastAsBigInt}, + {"uint32", reflect.TypeOf(uint32(0)), CastAsBigInt}, + {"uint64", reflect.TypeOf(uint64(0)), CastAsBigInt}, + {"float32", reflect.TypeOf(float32(0)), CastAsFloat}, + {"float64", reflect.TypeOf(float64(0)), CastAsFloat}, {"bool", reflect.TypeOf(false), CastAsBoolean}, + {"time.Time", reflect.TypeOf(time.Time{}), CastAsTimestamp}, + {"*time.Time", reflect.TypeOf((*time.Time)(nil)), CastAsTimestamp}, {"map", reflect.TypeOf(map[string]any{}), CastAsJSON}, {"slice", reflect.TypeOf([]string{}), CastAsJSON}, {"interface", reflect.TypeOf((*any)(nil)).Elem(), CastAsJSON}, - {"*string", reflect.TypeOf((*string)(nil)), CastAsString}, - {"*int", reflect.TypeOf((*int)(nil)), CastAsNumber}, + {"*string", reflect.TypeOf((*string)(nil)), CastAsText}, + {"*int", reflect.TypeOf((*int)(nil)), CastAsBigInt}, {"*bool", reflect.TypeOf((*bool)(nil)), CastAsBoolean}, } @@ -362,8 +365,8 @@ func TestTableSchemaFullStruct(t *testing.T) { if !ok { t.Fatal("missing email column") } - if *email.CastAs != CastAsString { - t.Errorf("email cast_as: got %q, want %q", *email.CastAs, CastAsString) + if *email.CastAs != CastAsText { + t.Errorf("email cast_as: got %q, want %q", *email.CastAs, CastAsText) } if email.Indexes == nil { t.Fatal("email indexes: expected non-nil") @@ -644,13 +647,13 @@ func TestTableSchemaPointerFieldTypeInference(t *testing.T) { } email := td.columns["email"] - if *email.CastAs != CastAsString { - t.Errorf("email cast_as: got %q, want %q", *email.CastAs, CastAsString) + if *email.CastAs != CastAsText { + t.Errorf("email cast_as: got %q, want %q", *email.CastAs, CastAsText) } age := td.columns["age"] - if *age.CastAs != CastAsNumber { - t.Errorf("age cast_as: got %q, want %q", *age.CastAs, CastAsNumber) + if *age.CastAs != CastAsBigInt { + t.Errorf("age cast_as: got %q, want %q", *age.CastAs, CastAsBigInt) } ok := td.columns["ok"] @@ -703,8 +706,9 @@ func TestBuildEncryptConfigJSONOutput(t *testing.T) { if !ok { t.Fatal("email column: not a map") } - if emailCol["cast_as"] != "string" { - t.Errorf("email cast_as: got %v, want %q", emailCol["cast_as"], "string") + // "string" is normalized to the canonical "text" on the wire. + if emailCol["cast_as"] != "text" { + t.Errorf("email cast_as: got %v, want %q", emailCol["cast_as"], "text") } emailIndexes, ok := emailCol["indexes"].(map[string]any) @@ -722,8 +726,9 @@ func TestBuildEncryptConfigJSONOutput(t *testing.T) { if !ok { t.Fatal("age column: not a map") } - if ageCol["cast_as"] != "number" { - t.Errorf("age cast_as: got %v, want %q", ageCol["cast_as"], "number") + // "number" is normalized to the canonical "float" on the wire. + if ageCol["cast_as"] != "float" { + t.Errorf("age cast_as: got %v, want %q", ageCol["cast_as"], "float") } ageIndexes, ok := ageCol["indexes"].(map[string]any) @@ -735,6 +740,61 @@ func TestBuildEncryptConfigJSONOutput(t *testing.T) { } } +// --- Canonicalization tests --- + +func TestBuildEncryptConfigInjectsSteVecArrayIndexMode(t *testing.T) { + t.Parallel() + + td := NewSchema("docs"). + Column("body", CastAsJSON).SearchableJSON("docs/body").Done(). + Build() + + config := buildEncryptConfigFromSchemas([]*TableDef{td}) + data, err := json.Marshal(config) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + + steVec := result["tables"].(map[string]any)["docs"].(map[string]any)["body"].(map[string]any)["indexes"].(map[string]any)["ste_vec"].(map[string]any) + if steVec["array_index_mode"] != "none" { + t.Errorf("array_index_mode: got %v, want %q", steVec["array_index_mode"], "none") + } + + // Canonicalization must not mutate the stored schema. + if td.columns["body"].Indexes.SteVecIndex.ArrayIndexMode != nil { + t.Error("stored schema ste_vec ArrayIndexMode should remain nil") + } +} + +func TestBuildEncryptConfigNormalizesBigInt(t *testing.T) { + t.Parallel() + + td := NewSchema("t"). + Column("n", CastAsBigInt).OrderAndRange().Done(). + Build() + + config := buildEncryptConfigFromSchemas([]*TableDef{td}) + data, err := json.Marshal(config) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + + col := result["tables"].(map[string]any)["t"].(map[string]any)["n"].(map[string]any) + if col["cast_as"] != "big_int" { + t.Errorf("cast_as: got %v, want %q", col["cast_as"], "big_int") + } +} + // --- Integration: full round-trip test --- func TestBuildEncryptConfigIntegration(t *testing.T) {