diff --git a/core/src/lib.rs b/core/src/lib.rs index 1e93e35..28c6828 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -237,12 +237,82 @@ impl TextEncoding { } fn decode_utf16(bytes: &[u8], conv: fn([u8; 2]) -> u16) -> String { - // A trailing odd byte (truncated UTF-16) is dropped by chunks_exact. - let units: Vec = bytes.chunks_exact(2).map(|c| conv([c[0], c[1]])).collect(); - String::from_utf16_lossy(&units) + // The DB-encoding path keeps its lossy-by-default contract: it discards + // the flag, so a truncated or corrupt unit still yields U+FFFD as before. + // The pairing itself lives in `decode_utf16_units` (DRY — the Local + // Storage decode reuses it and keeps the flag). + decode_utf16_units(bytes, conv).0 } } +/// Shared UTF-16 → `String` pairing: pairs 2-byte code units via `conv`, resolves +/// surrogate pairs, and reports whether the decode was **lossy**. A trailing odd +/// byte (half a code unit) or an unpaired surrogate emits U+FFFD and sets the +/// flag; it never panics or errors. Endianness is the caller's via `conv`. +fn decode_utf16_units(bytes: &[u8], conv: fn([u8; 2]) -> u16) -> (String, bool) { + // An odd trailing byte is half a code unit — real data was truncated. It is + // dropped by `chunks_exact`; the flag records that a byte was lost. + let mut lossy = bytes.len() % 2 != 0; + let units = bytes.chunks_exact(2).map(|c| conv([c[0], c[1]])); + let mut text = String::new(); + for unit in char::decode_utf16(units) { + if let Ok(c) = unit { + text.push(c); + } else { + lossy = true; + text.push(char::REPLACEMENT_CHARACTER); + } + } + (text, lossy) +} + +/// A WebKit/Chrome Local Storage `ItemTable.value` decoded to text, plus whether +/// the decode was lossy. `lossy` is a struct field, not a side-channel warning, +/// so a caller cannot render a lossy value as if it were faithfully recovered +/// (secure by design). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct LocalStorageValue { + /// The decoded string; any code unit that could not be decoded is a U+FFFD. + pub text: String, + /// `true` when at least one input byte/unit could not be decoded cleanly (an + /// odd-length BLOB or an unpaired surrogate). + pub lossy: bool, +} + +/// Decode a WebKit/Chromium Local Storage `ItemTable.value` BLOB to a `String`. +/// +/// A `.localstorage` file is a standard `SQLite` database this crate already +/// reads; the one artifact-specific quirk is that the `value` column is a BLOB +/// holding the string as raw **UTF-16 little-endian** code units — no BOM, no +/// type-prefix byte — so a normal dump surfaces it as opaque hex. This turns +/// such a BLOB back into readable text. +/// +/// Panic-free and lossy-by-report: an odd-length BLOB (a trailing half code +/// unit) or an unpaired surrogate yields U+FFFD and sets +/// [`LocalStorageValue::lossy`] rather than erroring or panicking. An empty BLOB +/// decodes to the empty string with `lossy == false`. +#[must_use] +pub fn decode_localstorage_value(blob: &[u8]) -> LocalStorageValue { + let (text, lossy) = decode_utf16_units(blob, u16::from_le_bytes); + LocalStorageValue { text, lossy } +} + +/// Recognize the WebKit/Chromium Local Storage `ItemTable(key TEXT, value BLOB)` +/// table, so a caller knows when [`decode_localstorage_value`] applies to a +/// dumped table's `value` column. +/// +/// Keyed on the distinctive table name `ItemTable` — the name WebKit/Chromium +/// create for Local Storage. The column names are deliberately NOT part of the +/// test: the real schema declares them with `ON CONFLICT` clauses +/// (`key TEXT UNIQUE ON CONFLICT REPLACE, value BLOB NOT NULL ON CONFLICT FAIL`) +/// that a lightweight `CREATE TABLE` parse does not always split cleanly, so a +/// name match is the robust signal. The row shape (a TEXT key, a BLOB value) +/// still surfaces positionally in each [`Row`]. +#[must_use] +pub fn is_local_storage_item_table(table_name: &str) -> bool { + table_name == "ItemTable" +} + /// Parsed 100-byte `SQLite` file header. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Header { @@ -5123,6 +5193,52 @@ mod tests { assert_eq!(n, 4); } + #[test] + fn localstorage_decodes_known_utf16le_bytes() { + // Independent oracle: these UTF-16-LE bytes are derived from the Unicode + // code points and the surrogate-pair formula, NOT from Rust's encoder, so + // a matching round-trip validates the decoder against the documented + // construction (Evidence-Based Rigor tier 2). + // 'A' U+0041 -> 41 00 + // '中' U+4E2D -> 2D 4E + // '😀' U+1F600 -> surrogate pair D83D DE00 -> 3D D8 00 DE + let bytes = [0x41, 0x00, 0x2D, 0x4E, 0x3D, 0xD8, 0x00, 0xDE]; + let out = decode_localstorage_value(&bytes); + assert_eq!(out.text, "A中😀"); + assert!(!out.lossy, "a fully-paired BLOB is not lossy"); + } + + #[test] + fn localstorage_empty_blob_is_empty_not_lossy() { + let out = decode_localstorage_value(&[]); + assert_eq!(out.text, ""); + assert!(!out.lossy); + } + + #[test] + fn localstorage_odd_length_blob_is_lossy_not_panic() { + // 'A' (41 00) then a lone trailing byte 42 — half a code unit was cut off. + let out = decode_localstorage_value(&[0x41, 0x00, 0x42]); + assert_eq!(out.text, "A"); + assert!(out.lossy, "a trailing half code unit is a lossy truncation"); + } + + #[test] + fn localstorage_lone_surrogate_is_replacement_and_lossy() { + // High surrogate D83D (LE 3D D8) with no following low surrogate. + let out = decode_localstorage_value(&[0x3D, 0xD8]); + assert_eq!(out.text, "\u{FFFD}"); + assert!(out.lossy); + } + + #[test] + fn item_table_schema_recognized_and_others_rejected() { + assert!(is_local_storage_item_table("ItemTable")); + assert!(!is_local_storage_item_table("moz_places")); + assert!(!is_local_storage_item_table("itemtable")); + assert!(!is_local_storage_item_table("")); + } + #[test] fn decode_value_int_literals() { assert_eq!( diff --git a/core/tests/localstorage_utf16_tests.rs b/core/tests/localstorage_utf16_tests.rs new file mode 100644 index 0000000..65c41f9 --- /dev/null +++ b/core/tests/localstorage_utf16_tests.rs @@ -0,0 +1,201 @@ +//! Real-artifact validation for the WebKit/Chrome Local Storage value-decode +//! convenience. A `.localstorage` file is a STANDARD SQLite database that this +//! crate already opens and dumps; the only artifact-specific quirk is that the +//! `ItemTable.value` column is a BLOB holding the string as **UTF-16-LE** (no +//! BOM, no type-prefix byte), so a plain dump surfaces it as opaque hex. +//! +//! The oracle is the real `sqlite3` engine: it builds the `.localstorage` file +//! and stores each value as a genuine UTF-16-LE BLOB, the native reader reads +//! the BLOB back, and [`decode_localstorage_value`] must recover the EXACT +//! original string — including a non-ASCII value that exercises surrogate pairs. +//! Gated on a `sqlite3` binary (PATH, or `SQLITE3_BIN`); skips gracefully when +//! absent so CI runners without `sqlite3` stay green (mirrors the +//! `rebuild_sqlite3_oracle` pattern). + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fmt::Write as _; +use std::path::PathBuf; +use std::process::Command; + +use sqlite_core::{decode_localstorage_value, is_local_storage_item_table, Database, Value}; + +/// Resolve a usable `sqlite3` binary, or `None` (test then skips). +fn sqlite3_bin() -> Option { + let bin = std::env::var("SQLITE3_BIN").unwrap_or_else(|_| "sqlite3".to_string()); + Command::new(&bin) + .arg("--version") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|_| bin) +} + +/// A unique temp path for a generated `.localstorage` file (caller cleans up). +fn temp_path(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + p.push(format!("sqlite4n6_localstorage_{tag}_{nonce}.localstorage")); + p +} + +/// Encode a string as the UTF-16-LE bytes `WebKit` stores, rendered as a +/// `sqlite3` `x'..'` BLOB hex literal. +fn utf16le_hex(s: &str) -> String { + let mut out = String::new(); + for byte in s.encode_utf16().flat_map(u16::to_le_bytes) { + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Run `sqlite3 ""`, asserting success. +fn run_sql(bin: &str, db: &PathBuf, sql: &str) { + let out = Command::new(bin) + .arg(db) + .arg(sql) + .output() + .expect("sqlite3 must execute"); + assert!( + out.status.success(), + "sqlite3 failed: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[test] +fn item_table_utf16le_blob_values_round_trip() { + let Some(bin) = sqlite3_bin() else { + eprintln!("SKIP item_table_utf16le_blob_values_round_trip: no sqlite3 (set SQLITE3_BIN)"); + return; + }; + + // The real WebKit/Chrome Local Storage schema, values stored as UTF-16-LE + // BLOBs. The non-ASCII cases (CJK + emoji) exercise multi-code-unit and + // surrogate-pair decoding, not just the ASCII fast path. + let cases = [ + ("token", "hello"), + ("accents", "héllo wörld"), + ("cjk", "中文字符串"), + ("emoji", "party 😀🎉 time"), + ]; + + let mut sql = String::from( + "CREATE TABLE ItemTable (key TEXT UNIQUE ON CONFLICT REPLACE, \ + value BLOB NOT NULL ON CONFLICT FAIL);", + ); + for (k, v) in cases { + let _ = write!( + sql, + "INSERT INTO ItemTable VALUES('{k}', x'{}');", + utf16le_hex(v) + ); + } + + let path = temp_path("roundtrip"); + run_sql(&bin, &path, &sql); + + let bytes = std::fs::read(&path).unwrap(); + let db = Database::open(bytes).expect("a .localstorage is a standard SQLite db"); + let dumps = db.live_table_rows(); + let item = dumps + .iter() + .find(|d| is_local_storage_item_table(&d.name)) + .expect("ItemTable schema recognized"); + + let mut recovered = std::collections::BTreeMap::new(); + for row in &item.rows { + let key = match &row.values[0] { + Value::Text(s) => s.clone(), + other => panic!("key column should be TEXT, got {other:?}"), + }; + let decoded = match &row.values[1] { + // SQLite sees the value as an opaque BLOB — this is the whole gap the + // convenience fills. + Value::Blob(b) => decode_localstorage_value(b), + other => panic!("value column should be a BLOB, got {other:?}"), + }; + assert!( + !decoded.lossy, + "a clean UTF-16-LE value must not be lossy: {key}" + ); + recovered.insert(key, decoded.text); + } + + for (k, v) in cases { + assert_eq!( + recovered.get(k).map(String::as_str), + Some(v), + "exact UTF-16-LE round-trip for key {k}" + ); + } + + let _ = std::fs::remove_file(&path); +} + +#[test] +fn item_table_null_value_reads_as_null_not_decoded() { + let Some(bin) = sqlite3_bin() else { + eprintln!( + "SKIP item_table_null_value_reads_as_null_not_decoded: no sqlite3 (set SQLITE3_BIN)" + ); + return; + }; + + // A NULL value is never a BLOB at the reader boundary, so the decode is + // never mis-applied to it (graceful by construction). SQLite may store the + // trailing NULL as `Value::Null` or omit it from the record entirely; both + // mean "no BLOB to decode", which is what the caller must be able to rely on. + let path = temp_path("null"); + run_sql( + &bin, + &path, + "CREATE TABLE ItemTable (key TEXT, value BLOB); \ + INSERT INTO ItemTable VALUES('present', x'68006900'); \ + INSERT INTO ItemTable VALUES('missing', NULL);", + ); + + let bytes = std::fs::read(&path).unwrap(); + let db = Database::open(bytes).unwrap(); + let dumps = db.live_table_rows(); + let item = dumps + .iter() + .find(|d| is_local_storage_item_table(&d.name)) + .expect("ItemTable schema recognized"); + + let mut present_decoded = false; + let mut missing_not_blob = false; + for row in &item.rows { + let key = match &row.values[0] { + Value::Text(s) => s.clone(), + other => panic!("key column should be TEXT, got {other:?}"), + }; + let value = row.values.get(1); + match key.as_str() { + "present" => match value { + Some(Value::Blob(b)) => { + let decoded = decode_localstorage_value(b); + assert_eq!(decoded.text, "hi"); + assert!(!decoded.lossy); + present_decoded = true; + } + other => panic!("present value should be a BLOB, got {other:?}"), + }, + "missing" => { + assert!( + !matches!(value, Some(Value::Blob(_))), + "a NULL value must not surface as a BLOB" + ); + missing_not_blob = true; + } + _ => {} + } + } + assert!(present_decoded, "the present BLOB value decoded"); + assert!(missing_not_blob, "the NULL value was not a decodable BLOB"); + + let _ = std::fs::remove_file(&path); +}