From 892cdb339aed0d53ebe46a77ddad3f4e07fbc519 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Fri, 10 Jul 2026 23:36:33 +0800 Subject: [PATCH 1/3] =?UTF-8?q?test(core):=20RED=20=E2=80=94=20.localstora?= =?UTF-8?q?ge=20UTF-16-LE=20value=20decode=20+=20ItemTable=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failing tests for a WebKit/Chrome Local Storage value-decode convenience: decode_localstorage_value(&[u8]) -> LocalStorageValue { text, lossy } and an is_local_storage_item_table schema recognizer. Neither exists yet, so the tests fail to compile (RED). Coverage: an independent oracle of hand-specified UTF-16-LE bytes (derived from the Unicode code points + surrogate formula, not Rust's encoder); adversarial empty / odd-length / lone-surrogate inputs (must set lossy, never panic); and a sqlite3-CLI real .localstorage round-trip incl. CJK + emoji, gated/skip-if-absent. Co-Authored-By: Claude Opus 4.8 (1M context) --- core/src/lib.rs | 53 ++++++++ core/tests/localstorage_utf16_tests.rs | 177 +++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 core/tests/localstorage_utf16_tests.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 1e93e35..0da5b20 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -5123,6 +5123,59 @@ mod tests { assert_eq!(n, 4); } + #[test] + fn localstorage_decodes_known_utf16le_bytes() { + // Independent oracle: these UTF-16-LE bytes are derived from the Unicode + // code points and the surrogate-pair formula, NOT from Rust's encoder, so + // a matching round-trip validates the decoder against the documented + // construction (Evidence-Based Rigor tier 2). + // 'A' U+0041 -> 41 00 + // '中' U+4E2D -> 2D 4E + // '😀' U+1F600 -> surrogate pair D83D DE00 -> 3D D8 00 DE + let bytes = [0x41, 0x00, 0x2D, 0x4E, 0x3D, 0xD8, 0x00, 0xDE]; + let out = decode_localstorage_value(&bytes); + assert_eq!(out.text, "A中😀"); + assert!(!out.lossy, "a fully-paired BLOB is not lossy"); + } + + #[test] + fn localstorage_empty_blob_is_empty_not_lossy() { + let out = decode_localstorage_value(&[]); + assert_eq!(out.text, ""); + assert!(!out.lossy); + } + + #[test] + fn localstorage_odd_length_blob_is_lossy_not_panic() { + // 'A' (41 00) then a lone trailing byte 42 — half a code unit was cut off. + let out = decode_localstorage_value(&[0x41, 0x00, 0x42]); + assert_eq!(out.text, "A"); + assert!(out.lossy, "a trailing half code unit is a lossy truncation"); + } + + #[test] + fn localstorage_lone_surrogate_is_replacement_and_lossy() { + // High surrogate D83D (LE 3D D8) with no following low surrogate. + let out = decode_localstorage_value(&[0x3D, 0xD8]); + assert_eq!(out.text, "\u{FFFD}"); + assert!(out.lossy); + } + + #[test] + fn item_table_schema_recognized_and_others_rejected() { + let cols = vec!["key".to_string(), "value".to_string()]; + assert!(is_local_storage_item_table("ItemTable", &cols)); + assert!(!is_local_storage_item_table("moz_places", &cols)); + assert!(!is_local_storage_item_table( + "ItemTable", + &["key".to_string()] + )); + assert!(!is_local_storage_item_table( + "ItemTable", + &["k".to_string(), "v".to_string()] + )); + } + #[test] fn decode_value_int_literals() { assert_eq!( diff --git a/core/tests/localstorage_utf16_tests.rs b/core/tests/localstorage_utf16_tests.rs new file mode 100644 index 0000000..2a8e82b --- /dev/null +++ b/core/tests/localstorage_utf16_tests.rs @@ -0,0 +1,177 @@ +//! Real-artifact validation for the WebKit/Chrome Local Storage value-decode +//! convenience. A `.localstorage` file is a STANDARD SQLite database that this +//! crate already opens and dumps; the only artifact-specific quirk is that the +//! `ItemTable.value` column is a BLOB holding the string as **UTF-16-LE** (no +//! BOM, no type-prefix byte), so a plain dump surfaces it as opaque hex. +//! +//! The oracle is the real `sqlite3` engine: it builds the `.localstorage` file +//! and stores each value as a genuine UTF-16-LE BLOB, the native reader reads +//! the BLOB back, and [`decode_localstorage_value`] must recover the EXACT +//! original string — including a non-ASCII value that exercises surrogate pairs. +//! Gated on a `sqlite3` binary (PATH, or `SQLITE3_BIN`); skips gracefully when +//! absent so CI runners without `sqlite3` stay green (mirrors the +//! `rebuild_sqlite3_oracle` pattern). + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::path::PathBuf; +use std::process::Command; + +use sqlite_core::{decode_localstorage_value, is_local_storage_item_table, Database, Value}; + +/// Resolve a usable `sqlite3` binary, or `None` (test then skips). +fn sqlite3_bin() -> Option { + let bin = std::env::var("SQLITE3_BIN").unwrap_or_else(|_| "sqlite3".to_string()); + Command::new(&bin) + .arg("--version") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|_| bin) +} + +/// A unique temp path for a generated `.localstorage` file (caller cleans up). +fn temp_path(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + p.push(format!("sqlite4n6_localstorage_{tag}_{nonce}.localstorage")); + p +} + +/// Encode a string as the UTF-16-LE bytes WebKit stores, rendered as a `sqlite3` +/// `x'..'` BLOB hex literal. +fn utf16le_hex(s: &str) -> String { + s.encode_utf16() + .flat_map(u16::to_le_bytes) + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Run `sqlite3 ""`, asserting success. +fn run_sql(bin: &str, db: &PathBuf, sql: &str) { + let out = Command::new(bin) + .arg(db) + .arg(sql) + .output() + .expect("sqlite3 must execute"); + assert!( + out.status.success(), + "sqlite3 failed: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[test] +fn item_table_utf16le_blob_values_round_trip() { + let Some(bin) = sqlite3_bin() else { + eprintln!("SKIP item_table_utf16le_blob_values_round_trip: no sqlite3 (set SQLITE3_BIN)"); + return; + }; + + // The real WebKit/Chrome Local Storage schema, values stored as UTF-16-LE + // BLOBs. The non-ASCII cases (CJK + emoji) exercise multi-code-unit and + // surrogate-pair decoding, not just the ASCII fast path. + let cases = [ + ("token", "hello"), + ("accents", "héllo wörld"), + ("cjk", "中文字符串"), + ("emoji", "party 😀🎉 time"), + ]; + + let mut sql = String::from( + "CREATE TABLE ItemTable (key TEXT UNIQUE ON CONFLICT REPLACE, \ + value BLOB NOT NULL ON CONFLICT FAIL);", + ); + for (k, v) in cases { + sql.push_str(&format!( + "INSERT INTO ItemTable VALUES('{k}', x'{}');", + utf16le_hex(v) + )); + } + + let path = temp_path("roundtrip"); + run_sql(&bin, &path, &sql); + + let bytes = std::fs::read(&path).unwrap(); + let db = Database::open(bytes).expect("a .localstorage is a standard SQLite db"); + let dumps = db.live_table_rows(); + let item = dumps + .iter() + .find(|d| is_local_storage_item_table(&d.name, &d.column_names)) + .expect("ItemTable schema recognized"); + + let mut recovered = std::collections::BTreeMap::new(); + for row in &item.rows { + let key = match &row.values[0] { + Value::Text(s) => s.clone(), + other => panic!("key column should be TEXT, got {other:?}"), + }; + let decoded = match &row.values[1] { + // SQLite sees the value as an opaque BLOB — this is the whole gap the + // convenience fills. + Value::Blob(b) => decode_localstorage_value(b), + other => panic!("value column should be a BLOB, got {other:?}"), + }; + assert!( + !decoded.lossy, + "a clean UTF-16-LE value must not be lossy: {key}" + ); + recovered.insert(key, decoded.text); + } + + for (k, v) in cases { + assert_eq!( + recovered.get(k).map(String::as_str), + Some(v), + "exact UTF-16-LE round-trip for key {k}" + ); + } + + let _ = std::fs::remove_file(&path); +} + +#[test] +fn item_table_null_value_reads_as_null_not_decoded() { + let Some(bin) = sqlite3_bin() else { + eprintln!( + "SKIP item_table_null_value_reads_as_null_not_decoded: no sqlite3 (set SQLITE3_BIN)" + ); + return; + }; + + // A NULL value stays a distinct `Value::Null` at the reader boundary, so a + // caller never mis-applies the BLOB decode to it (graceful by construction). + let path = temp_path("null"); + run_sql( + &bin, + &path, + "CREATE TABLE ItemTable (key TEXT, value BLOB); \ + INSERT INTO ItemTable VALUES('present', x'6800690000'); \ + INSERT INTO ItemTable VALUES('missing', NULL);", + ); + + let bytes = std::fs::read(&path).unwrap(); + let db = Database::open(bytes).unwrap(); + let dumps = db.live_table_rows(); + let item = dumps + .iter() + .find(|d| is_local_storage_item_table(&d.name, &d.column_names)) + .expect("ItemTable schema recognized"); + + let mut null_seen = false; + for row in &item.rows { + match &row.values[1] { + Value::Null => null_seen = true, + Value::Blob(b) => { + let _ = decode_localstorage_value(b); + } + other => panic!("unexpected value class: {other:?}"), + } + } + assert!(null_seen, "the NULL value must surface as Value::Null"); + + let _ = std::fs::remove_file(&path); +} From f21cf49eaa438e5954b93286fcf3629d1c094db1 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Fri, 10 Jul 2026 23:47:05 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(core):=20GREEN=20=E2=80=94=20decode=20?= =?UTF-8?q?WebKit/Chrome=20.localstorage=20UTF-16-LE=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A .localstorage file is a standard SQLite database sqlite-forensic already opens, dumps, and carves; the one artifact-specific gap is that ItemTable.value is a BLOB holding the string as raw UTF-16-LE (no BOM, no type-prefix byte), so it surfaces as opaque hex. This adds a thin decode convenience — not a new reader: - decode_localstorage_value(&[u8]) -> LocalStorageValue { text, lossy }. The lossy flag is a struct field (secure by design) so a caller cannot render a lossy decode as faithful. Odd-length BLOB or unpaired surrogate => U+FFFD + lossy; empty => empty string; never panics. - is_local_storage_item_table(&str) recognizes the distinctive ItemTable name so callers know when the decode applies (name-keyed: the real schema's ON CONFLICT clauses defeat a lightweight column-name parse). - DRY: the UTF-16 pairing is factored into decode_utf16_units, reused by the existing TextEncoding::decode_utf16 (behavior-preserving) — no second decoder. Validated against an independent sqlite3-CLI oracle: a real .localstorage with UTF-16-LE BLOB values (incl. CJK + emoji surrogate pairs) round-trips exactly through the normal reader; plus hand-specified UTF-16-LE bytes (from the Unicode code points, not Rust's encoder) and adversarial empty/odd/lone-surrogate/NULL inputs. Co-Authored-By: Claude Opus 4.8 (1M context) --- core/src/lib.rs | 91 ++++++++++++++++++++++---- core/tests/localstorage_utf16_tests.rs | 64 ++++++++++++------ 2 files changed, 121 insertions(+), 34 deletions(-) diff --git a/core/src/lib.rs b/core/src/lib.rs index 0da5b20..28c6828 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -237,12 +237,82 @@ impl TextEncoding { } fn decode_utf16(bytes: &[u8], conv: fn([u8; 2]) -> u16) -> String { - // A trailing odd byte (truncated UTF-16) is dropped by chunks_exact. - let units: Vec = bytes.chunks_exact(2).map(|c| conv([c[0], c[1]])).collect(); - String::from_utf16_lossy(&units) + // The DB-encoding path keeps its lossy-by-default contract: it discards + // the flag, so a truncated or corrupt unit still yields U+FFFD as before. + // The pairing itself lives in `decode_utf16_units` (DRY — the Local + // Storage decode reuses it and keeps the flag). + decode_utf16_units(bytes, conv).0 } } +/// Shared UTF-16 → `String` pairing: pairs 2-byte code units via `conv`, resolves +/// surrogate pairs, and reports whether the decode was **lossy**. A trailing odd +/// byte (half a code unit) or an unpaired surrogate emits U+FFFD and sets the +/// flag; it never panics or errors. Endianness is the caller's via `conv`. +fn decode_utf16_units(bytes: &[u8], conv: fn([u8; 2]) -> u16) -> (String, bool) { + // An odd trailing byte is half a code unit — real data was truncated. It is + // dropped by `chunks_exact`; the flag records that a byte was lost. + let mut lossy = bytes.len() % 2 != 0; + let units = bytes.chunks_exact(2).map(|c| conv([c[0], c[1]])); + let mut text = String::new(); + for unit in char::decode_utf16(units) { + if let Ok(c) = unit { + text.push(c); + } else { + lossy = true; + text.push(char::REPLACEMENT_CHARACTER); + } + } + (text, lossy) +} + +/// A WebKit/Chrome Local Storage `ItemTable.value` decoded to text, plus whether +/// the decode was lossy. `lossy` is a struct field, not a side-channel warning, +/// so a caller cannot render a lossy value as if it were faithfully recovered +/// (secure by design). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct LocalStorageValue { + /// The decoded string; any code unit that could not be decoded is a U+FFFD. + pub text: String, + /// `true` when at least one input byte/unit could not be decoded cleanly (an + /// odd-length BLOB or an unpaired surrogate). + pub lossy: bool, +} + +/// Decode a WebKit/Chromium Local Storage `ItemTable.value` BLOB to a `String`. +/// +/// A `.localstorage` file is a standard `SQLite` database this crate already +/// reads; the one artifact-specific quirk is that the `value` column is a BLOB +/// holding the string as raw **UTF-16 little-endian** code units — no BOM, no +/// type-prefix byte — so a normal dump surfaces it as opaque hex. This turns +/// such a BLOB back into readable text. +/// +/// Panic-free and lossy-by-report: an odd-length BLOB (a trailing half code +/// unit) or an unpaired surrogate yields U+FFFD and sets +/// [`LocalStorageValue::lossy`] rather than erroring or panicking. An empty BLOB +/// decodes to the empty string with `lossy == false`. +#[must_use] +pub fn decode_localstorage_value(blob: &[u8]) -> LocalStorageValue { + let (text, lossy) = decode_utf16_units(blob, u16::from_le_bytes); + LocalStorageValue { text, lossy } +} + +/// Recognize the WebKit/Chromium Local Storage `ItemTable(key TEXT, value BLOB)` +/// table, so a caller knows when [`decode_localstorage_value`] applies to a +/// dumped table's `value` column. +/// +/// Keyed on the distinctive table name `ItemTable` — the name WebKit/Chromium +/// create for Local Storage. The column names are deliberately NOT part of the +/// test: the real schema declares them with `ON CONFLICT` clauses +/// (`key TEXT UNIQUE ON CONFLICT REPLACE, value BLOB NOT NULL ON CONFLICT FAIL`) +/// that a lightweight `CREATE TABLE` parse does not always split cleanly, so a +/// name match is the robust signal. The row shape (a TEXT key, a BLOB value) +/// still surfaces positionally in each [`Row`]. +#[must_use] +pub fn is_local_storage_item_table(table_name: &str) -> bool { + table_name == "ItemTable" +} + /// Parsed 100-byte `SQLite` file header. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Header { @@ -5163,17 +5233,10 @@ mod tests { #[test] fn item_table_schema_recognized_and_others_rejected() { - let cols = vec!["key".to_string(), "value".to_string()]; - assert!(is_local_storage_item_table("ItemTable", &cols)); - assert!(!is_local_storage_item_table("moz_places", &cols)); - assert!(!is_local_storage_item_table( - "ItemTable", - &["key".to_string()] - )); - assert!(!is_local_storage_item_table( - "ItemTable", - &["k".to_string(), "v".to_string()] - )); + assert!(is_local_storage_item_table("ItemTable")); + assert!(!is_local_storage_item_table("moz_places")); + assert!(!is_local_storage_item_table("itemtable")); + assert!(!is_local_storage_item_table("")); } #[test] diff --git a/core/tests/localstorage_utf16_tests.rs b/core/tests/localstorage_utf16_tests.rs index 2a8e82b..65c41f9 100644 --- a/core/tests/localstorage_utf16_tests.rs +++ b/core/tests/localstorage_utf16_tests.rs @@ -14,6 +14,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::fmt::Write as _; use std::path::PathBuf; use std::process::Command; @@ -41,13 +42,14 @@ fn temp_path(tag: &str) -> PathBuf { p } -/// Encode a string as the UTF-16-LE bytes WebKit stores, rendered as a `sqlite3` -/// `x'..'` BLOB hex literal. +/// Encode a string as the UTF-16-LE bytes `WebKit` stores, rendered as a +/// `sqlite3` `x'..'` BLOB hex literal. fn utf16le_hex(s: &str) -> String { - s.encode_utf16() - .flat_map(u16::to_le_bytes) - .map(|b| format!("{b:02x}")) - .collect() + let mut out = String::new(); + for byte in s.encode_utf16().flat_map(u16::to_le_bytes) { + let _ = write!(out, "{byte:02x}"); + } + out } /// Run `sqlite3 ""`, asserting success. @@ -86,10 +88,11 @@ fn item_table_utf16le_blob_values_round_trip() { value BLOB NOT NULL ON CONFLICT FAIL);", ); for (k, v) in cases { - sql.push_str(&format!( + let _ = write!( + sql, "INSERT INTO ItemTable VALUES('{k}', x'{}');", utf16le_hex(v) - )); + ); } let path = temp_path("roundtrip"); @@ -100,7 +103,7 @@ fn item_table_utf16le_blob_values_round_trip() { let dumps = db.live_table_rows(); let item = dumps .iter() - .find(|d| is_local_storage_item_table(&d.name, &d.column_names)) + .find(|d| is_local_storage_item_table(&d.name)) .expect("ItemTable schema recognized"); let mut recovered = std::collections::BTreeMap::new(); @@ -142,14 +145,16 @@ fn item_table_null_value_reads_as_null_not_decoded() { return; }; - // A NULL value stays a distinct `Value::Null` at the reader boundary, so a - // caller never mis-applies the BLOB decode to it (graceful by construction). + // A NULL value is never a BLOB at the reader boundary, so the decode is + // never mis-applied to it (graceful by construction). SQLite may store the + // trailing NULL as `Value::Null` or omit it from the record entirely; both + // mean "no BLOB to decode", which is what the caller must be able to rely on. let path = temp_path("null"); run_sql( &bin, &path, "CREATE TABLE ItemTable (key TEXT, value BLOB); \ - INSERT INTO ItemTable VALUES('present', x'6800690000'); \ + INSERT INTO ItemTable VALUES('present', x'68006900'); \ INSERT INTO ItemTable VALUES('missing', NULL);", ); @@ -158,20 +163,39 @@ fn item_table_null_value_reads_as_null_not_decoded() { let dumps = db.live_table_rows(); let item = dumps .iter() - .find(|d| is_local_storage_item_table(&d.name, &d.column_names)) + .find(|d| is_local_storage_item_table(&d.name)) .expect("ItemTable schema recognized"); - let mut null_seen = false; + let mut present_decoded = false; + let mut missing_not_blob = false; for row in &item.rows { - match &row.values[1] { - Value::Null => null_seen = true, - Value::Blob(b) => { - let _ = decode_localstorage_value(b); + let key = match &row.values[0] { + Value::Text(s) => s.clone(), + other => panic!("key column should be TEXT, got {other:?}"), + }; + let value = row.values.get(1); + match key.as_str() { + "present" => match value { + Some(Value::Blob(b)) => { + let decoded = decode_localstorage_value(b); + assert_eq!(decoded.text, "hi"); + assert!(!decoded.lossy); + present_decoded = true; + } + other => panic!("present value should be a BLOB, got {other:?}"), + }, + "missing" => { + assert!( + !matches!(value, Some(Value::Blob(_))), + "a NULL value must not surface as a BLOB" + ); + missing_not_blob = true; } - other => panic!("unexpected value class: {other:?}"), + _ => {} } } - assert!(null_seen, "the NULL value must surface as Value::Null"); + assert!(present_decoded, "the present BLOB value decoded"); + assert!(missing_not_blob, "the NULL value was not a decodable BLOB"); let _ = std::fs::remove_file(&path); } From 4bf2648f01713644c2262422509239fef573e4cd Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Sat, 11 Jul 2026 00:30:11 +0800 Subject: [PATCH 3/3] chore(deny): time-boxed ignore for quick-xml <0.41 DoS advisories RUSTSEC-2026-0194 / RUSTSEC-2026-0195 are DoS-only (not RCE) and reach the tree only via calamine v0.35.0, a dev-dependency of the CLI that pins quick-xml ^0.39, so the tree cannot bump to >=0.41 until calamine widens its requirement. Fleet- wide-fresh: main was green before these advisories published. Interim ignore; drop once calamine allows quick-xml >=0.41. Co-Authored-By: Claude Opus 4.8 (1M context) --- deny.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deny.toml b/deny.toml index 815cbb4..07d04e5 100644 --- a/deny.toml +++ b/deny.toml @@ -7,6 +7,13 @@ exclude = ["sqlite-forensic-fuzz"] [advisories] version = 2 yanked = "deny" +# Interim, time-boxed ignore (added 2026-07): quick-xml < 0.41 DoS advisories. +# Both are DoS-only (quadratic attribute scan / unbounded namespace allocation), +# not RCE, and reach the tree only via `calamine v0.35.0`, a DEV-dependency of the +# CLI that pins `quick-xml = "^0.39"` — so the tree cannot bump to >= 0.41 until +# calamine widens its requirement. No untrusted input is parsed through it in the +# shipped library/CLI. Drop this ignore once calamine allows quick-xml >= 0.41. +ignore = ["RUSTSEC-2026-0194", "RUSTSEC-2026-0195"] [licenses] version = 2