From 61f7af8f1ecd0757435ef294c20f3af3e12e3c1c Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Tue, 17 Mar 2026 17:08:12 +0100 Subject: [PATCH 01/21] feat(evfs): add streaming segment format with chunk layout Add chunk_count field to SegmentEntry for streaming segments, streaming_segment_size/streaming_chunk_count layout functions, VaultChunkAad type alias, and derive_chunk_nonce helper. --- rust/src/api/evfs/mod.rs | 1 + rust/src/core/evfs/format.rs | 177 +++++++++++++++++++++++++++++++++- rust/src/core/evfs/segment.rs | 78 +++++++++++++++ 3 files changed, 251 insertions(+), 5 deletions(-) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index 29a3f48..d5e8d51 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -287,6 +287,7 @@ pub fn vault_write( gen, checksum, effective_algo, + 0, // monolithic (one-shot) segment )?; handle.index.add(entry)?; diff --git a/rust/src/core/evfs/format.rs b/rust/src/core/evfs/format.rs index 047cdba..0b9a528 100644 --- a/rust/src/core/evfs/format.rs +++ b/rust/src/core/evfs/format.rs @@ -72,6 +72,32 @@ pub fn total_vault_size(capacity: u64, index_pad_size: usize) -> Result u64 { + use crate::core::streaming::{CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE}; + if plaintext_size == 0 { + return 0; + } + let chunk_count = plaintext_size.div_ceil(CHUNK_SIZE as u64); + chunk_count * ENCRYPTED_CHUNK_SIZE as u64 +} + +/// Compute the number of chunks needed for a given plaintext size. +pub fn streaming_chunk_count(plaintext_size: u64) -> u32 { + use crate::core::streaming::CHUNK_SIZE; + if plaintext_size == 0 { + return 0; + } + plaintext_size.div_ceil(CHUNK_SIZE as u64) as u32 +} + // --------------------------------------------------------------------------- // VaultHeader // --------------------------------------------------------------------------- @@ -179,6 +205,9 @@ pub struct SegmentEntry { pub checksum: [u8; 32], /// Algorithm used to compress this segment. pub compression: CompressionAlgorithm, + /// Number of chunks for streaming segments. 0 = monolithic (one-shot), + /// >0 = N independently-encrypted chunks within this segment slot. + pub chunk_count: u32, } impl SegmentEntry { @@ -189,6 +218,7 @@ impl SegmentEntry { generation: u64, checksum: [u8; 32], compression: CompressionAlgorithm, + chunk_count: u32, ) -> Result { if name.is_empty() || name.len() > MAX_SEGMENT_NAME_LEN { return Err(CryptoError::InvalidParameter(format!( @@ -203,8 +233,14 @@ impl SegmentEntry { generation, checksum, compression, + chunk_count, }) } + + /// Returns true if this segment uses streaming (chunked) storage. + pub fn is_streaming(&self) -> bool { + self.chunk_count > 0 + } } // --------------------------------------------------------------------------- @@ -349,6 +385,7 @@ impl SegmentIndex { put_u64(&mut buf, entry.generation); buf.extend_from_slice(&entry.checksum); buf.push(entry.compression.to_u8()); + put_u32(&mut buf, entry.chunk_count); } for region in &self.free_regions { @@ -377,9 +414,9 @@ impl SegmentIndex { let entry_count = read_u32(data, &mut off)? as usize; let free_count = read_u32(data, &mut off)? as usize; - // Sanity-cap: the smallest possible entry is ~59 bytes (1-byte name), + // Sanity-cap: the smallest possible entry is ~63 bytes (1-byte name), // free region is 16 bytes. Reject clearly corrupted counts early. - let max_entries = data.len() / 59; + let max_entries = data.len() / 63; let max_free = data.len() / 16; if entry_count > max_entries { return Err(CryptoError::VaultCorrupted(format!( @@ -416,6 +453,7 @@ impl SegmentIndex { checksum.copy_from_slice(&checksum_bytes); let comp_byte = read_bytes(data, &mut off, 1)?; let compression = CompressionAlgorithm::from_u8(comp_byte[0])?; + let chunk_count = read_u32(data, &mut off)?; entries.push(SegmentEntry { name, offset, @@ -423,6 +461,7 @@ impl SegmentIndex { generation, checksum, compression, + chunk_count, }); } @@ -711,6 +750,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ); assert!(e.is_ok()); @@ -723,6 +763,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ); assert!(e.is_ok()); } @@ -737,13 +778,14 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ); assert!(e.is_err()); } #[test] fn test_segment_entry_name_empty() { - let e = SegmentEntry::new("", 0, 100, 0, dummy_checksum(0), CompressionAlgorithm::None); + let e = SegmentEntry::new("", 0, 100, 0, dummy_checksum(0), CompressionAlgorithm::None, 0); assert!(e.is_err()); } @@ -759,6 +801,7 @@ mod tests { 1, dummy_checksum(0xAA), CompressionAlgorithm::Zstd, + 0, // monolithic ) .expect("entry"), ) @@ -771,6 +814,7 @@ mod tests { 2, dummy_checksum(0xBB), CompressionAlgorithm::None, + 0, ) .expect("entry"), ) @@ -797,9 +841,11 @@ mod tests { assert_eq!(parsed.entries[0].generation, 1); assert_eq!(parsed.entries[0].checksum, dummy_checksum(0xAA)); assert_eq!(parsed.entries[0].compression, CompressionAlgorithm::Zstd); + assert_eq!(parsed.entries[0].chunk_count, 0); assert_eq!(parsed.entries[1].name, "photo.jpg"); assert_eq!(parsed.entries[1].compression, CompressionAlgorithm::None); + assert_eq!(parsed.entries[1].chunk_count, 0); assert_eq!(parsed.free_regions.len(), 1); assert_eq!(parsed.free_regions[0].offset, 12288); @@ -822,6 +868,7 @@ mod tests { 41, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -850,6 +897,7 @@ mod tests { 0, dummy_checksum(i as u8), *algo, + 0, ) .expect("entry"), ); @@ -920,8 +968,8 @@ mod tests { #[test] fn test_segment_index_overflow_min_pad() { let mut idx = SegmentIndex::new(512 * 1024); // 512KB → MIN_INDEX_PAD_SIZE - // Each entry with a 255-byte name uses 2 + 255 + 8 + 8 + 8 + 32 + 1 = 314 bytes. - // Index header is 32 bytes. (65536 - 32) / 314 ≈ 208 entries max. + // Each entry with a 255-byte name uses 2 + 255 + 8 + 8 + 8 + 32 + 1 + 4 = 318 bytes. + // Index header is 32 bytes. (65536 - 32) / 318 ≈ 205 entries max. for i in 0..210 { let name = format!("{:0>255}", i); idx.entries.push( @@ -932,6 +980,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1226,6 +1275,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"); let e2 = SegmentEntry::new( @@ -1235,6 +1285,7 @@ mod tests { 1, dummy_checksum(1), CompressionAlgorithm::None, + 0, ) .expect("entry"); idx.add(e1).expect("first add"); @@ -1255,6 +1306,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1266,6 +1318,7 @@ mod tests { 1, dummy_checksum(1), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1287,6 +1340,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1298,6 +1352,7 @@ mod tests { 1, dummy_checksum(1), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1326,6 +1381,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1353,6 +1409,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1364,6 +1421,7 @@ mod tests { 1, dummy_checksum(1), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1375,6 +1433,7 @@ mod tests { 2, dummy_checksum(2), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1420,6 +1479,7 @@ mod tests { 0, dummy_checksum(0), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1431,6 +1491,7 @@ mod tests { 1, dummy_checksum(1), CompressionAlgorithm::None, + 0, ) .expect("entry"), ); @@ -1466,4 +1527,110 @@ mod tests { }); assert!(idx.needs_defrag()); } + + // -- Streaming segment format ------------------------------------------- + + #[test] + fn test_segment_entry_chunk_count_roundtrip() { + let mut idx = SegmentIndex::new(1024 * 1024); + // Monolithic segment + idx.entries.push( + SegmentEntry::new( + "mono.bin", + 0, + 1024, + 0, + dummy_checksum(0), + CompressionAlgorithm::None, + 0, + ) + .expect("entry"), + ); + // Streaming segment with 5 chunks + idx.entries.push( + SegmentEntry::new( + "stream.bin", + 1024, + 5 * 65564, // 5 * ENCRYPTED_CHUNK_SIZE + 1, + dummy_checksum(1), + CompressionAlgorithm::None, + 5, + ) + .expect("entry"), + ); + + let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); + + assert_eq!(parsed.entries[0].chunk_count, 0); + assert!(!parsed.entries[0].is_streaming()); + assert_eq!(parsed.entries[1].chunk_count, 5); + assert!(parsed.entries[1].is_streaming()); + } + + #[test] + fn test_streaming_segment_size_zero() { + assert_eq!(streaming_segment_size(0), 0); + } + + #[test] + fn test_streaming_segment_size_single_chunk() { + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + // 1 byte → 1 chunk + assert_eq!(streaming_segment_size(1), ENCRYPTED_CHUNK_SIZE as u64); + // Exactly one chunk + assert_eq!(streaming_segment_size(64 * 1024), ENCRYPTED_CHUNK_SIZE as u64); + } + + #[test] + fn test_streaming_segment_size_multiple_chunks() { + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + // 64KB + 1 byte → 2 chunks + assert_eq!( + streaming_segment_size(64 * 1024 + 1), + 2 * ENCRYPTED_CHUNK_SIZE as u64 + ); + // 5 full chunks + assert_eq!( + streaming_segment_size(5 * 64 * 1024), + 5 * ENCRYPTED_CHUNK_SIZE as u64 + ); + } + + #[test] + fn test_streaming_chunk_count_values() { + assert_eq!(streaming_chunk_count(0), 0); + assert_eq!(streaming_chunk_count(1), 1); + assert_eq!(streaming_chunk_count(64 * 1024), 1); + assert_eq!(streaming_chunk_count(64 * 1024 + 1), 2); + assert_eq!(streaming_chunk_count(5 * 64 * 1024), 5); + } + + #[test] + fn test_is_streaming_helper() { + let mono = SegmentEntry::new( + "mono", + 0, + 100, + 0, + dummy_checksum(0), + CompressionAlgorithm::None, + 0, + ) + .expect("entry"); + assert!(!mono.is_streaming()); + + let stream = SegmentEntry::new( + "stream", + 0, + 65564, + 0, + dummy_checksum(0), + CompressionAlgorithm::None, + 1, + ) + .expect("entry"); + assert!(stream.is_streaming()); + } } diff --git a/rust/src/core/evfs/segment.rs b/rust/src/core/evfs/segment.rs index 48dd3c0..ae05d07 100644 --- a/rust/src/core/evfs/segment.rs +++ b/rust/src/core/evfs/segment.rs @@ -99,6 +99,27 @@ pub fn derive_segment_nonce( Ok(nonce) } +// --------------------------------------------------------------------------- +// Streaming segment support (per-chunk nonce + AAD) +// --------------------------------------------------------------------------- + +/// Per-chunk AAD for vault streaming segments. +/// Wire-compatible with the standalone streaming format's `ChunkAad`. +pub type VaultChunkAad = crate::core::streaming::ChunkAad; + +/// Derive a unique nonce for a specific chunk within a streaming segment. +/// +/// Reuses the existing HKDF-based nonce derivation with `chunk_index` as the +/// index parameter and the segment's `generation` counter, producing unique +/// nonces per chunk and per overwrite. +pub fn derive_chunk_nonce( + nonce_key: &[u8], + chunk_index: u64, + generation: u64, +) -> Result, CryptoError> { + derive_segment_nonce(nonce_key, chunk_index, generation, NONCE_LEN) +} + // --------------------------------------------------------------------------- // AEAD helpers (algorithm dispatch) // --------------------------------------------------------------------------- @@ -908,4 +929,61 @@ mod tests { file.read_exact(&mut after).expect("read"); assert_eq!(after, vec![0xAA; 256]); } + + // -- Chunk nonce derivation (streaming segments) ------------------------ + + #[test] + fn test_chunk_nonce_deterministic() { + let keys = derive_vault_keys(&test_master_key()).expect("derive"); + let n1 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 0, 0).expect("nonce"); + let n2 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 0, 0).expect("nonce"); + assert_eq!(n1, n2); + assert_eq!(n1.len(), NONCE_LEN); + } + + #[test] + fn test_chunk_nonce_unique_per_chunk() { + let keys = derive_vault_keys(&test_master_key()).expect("derive"); + let n0 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 0, 0).expect("nonce"); + let n1 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 1, 0).expect("nonce"); + let n2 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 2, 0).expect("nonce"); + assert_ne!(n0, n1); + assert_ne!(n1, n2); + assert_ne!(n0, n2); + } + + #[test] + fn test_chunk_nonce_unique_per_generation() { + let keys = derive_vault_keys(&test_master_key()).expect("derive"); + let n_gen0 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 0, 0).expect("nonce"); + let n_gen1 = derive_chunk_nonce(keys.nonce_key.as_bytes(), 0, 1).expect("nonce"); + assert_ne!(n_gen0, n_gen1); + } + + #[test] + fn test_chunk_nonce_matches_segment_nonce() { + // derive_chunk_nonce(key, chunk_idx, gen) must equal + // derive_segment_nonce(key, chunk_idx, gen, NONCE_LEN) + let keys = derive_vault_keys(&test_master_key()).expect("derive"); + let chunk = derive_chunk_nonce(keys.nonce_key.as_bytes(), 42, 7).expect("chunk"); + let segment = + derive_segment_nonce(keys.nonce_key.as_bytes(), 42, 7, NONCE_LEN).expect("segment"); + assert_eq!(chunk, segment); + } + + #[test] + fn test_vault_chunk_aad_wire_compat() { + // VaultChunkAad is a type alias for ChunkAad — verify wire format + use crate::core::streaming::AAD_SIZE; + let aad = VaultChunkAad { + index: 99, + is_final: true, + }; + let bytes = aad.to_bytes(); + assert_eq!(bytes.len(), AAD_SIZE); + // index = 99 LE + assert_eq!(u64::from_le_bytes(bytes[0..8].try_into().unwrap()), 99); + // is_final = true + assert_eq!(bytes[8], 1); + } } From 73b4a87ad6bedba02378dd99ed6f6d2874b2e231 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Tue, 17 Mar 2026 17:49:52 +0100 Subject: [PATCH 02/21] fix(evfs): harden streaming segment format for production - Add domain-separated HKDF info for chunk nonces (0x01 prefix) to prevent collisions with monolithic segment nonces - Make VaultChunkAad a standalone struct with generation field to bind chunks to their segment write (cross-segment splice defense) - Use checked_mul/try_from in streaming_segment_size/chunk_count to prevent silent integer overflow on pathological inputs - Validate chunk_count * ENCRYPTED_CHUNK_SIZE == size on index deserialization to reject corrupted/malicious indices - Unify strip_last_chunk_padding error messages (padding oracle defense) - Reject non-canonical is_final bytes in ChunkAad::from_bytes - Fix sanity cap and wire format doc comment for chunk_count field --- rust/src/core/evfs/format.rs | 114 ++++++++++++++++++++++++++++------ rust/src/core/evfs/segment.rs | 84 +++++++++++++++++++------ rust/src/core/streaming.rs | 35 ++++++++--- 3 files changed, 187 insertions(+), 46 deletions(-) diff --git a/rust/src/core/evfs/format.rs b/rust/src/core/evfs/format.rs index 0b9a528..3ba90a5 100644 --- a/rust/src/core/evfs/format.rs +++ b/rust/src/core/evfs/format.rs @@ -80,22 +80,33 @@ pub fn total_vault_size(capacity: u64, index_pad_size: usize) -> Result u64 { +/// +/// Returns `Err` if the result overflows `u64`. +pub fn streaming_segment_size(plaintext_size: u64) -> Result { use crate::core::streaming::{CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE}; if plaintext_size == 0 { - return 0; + return Ok(0); } let chunk_count = plaintext_size.div_ceil(CHUNK_SIZE as u64); - chunk_count * ENCRYPTED_CHUNK_SIZE as u64 + chunk_count + .checked_mul(ENCRYPTED_CHUNK_SIZE as u64) + .ok_or_else(|| { + CryptoError::InvalidParameter("streaming segment size overflows u64".into()) + }) } /// Compute the number of chunks needed for a given plaintext size. -pub fn streaming_chunk_count(plaintext_size: u64) -> u32 { +/// +/// Returns `Err` if the count exceeds `u32::MAX`. +pub fn streaming_chunk_count(plaintext_size: u64) -> Result { use crate::core::streaming::CHUNK_SIZE; if plaintext_size == 0 { - return 0; + return Ok(0); } - plaintext_size.div_ceil(CHUNK_SIZE as u64) as u32 + let count = plaintext_size.div_ceil(CHUNK_SIZE as u64); + u32::try_from(count).map_err(|_| { + CryptoError::InvalidParameter("streaming chunk count exceeds u32::MAX".into()) + }) } // --------------------------------------------------------------------------- @@ -353,7 +364,7 @@ impl SegmentIndex { /// -- entries -- /// per entry: /// [name_len: u16] [name: UTF-8] [offset: u64] [size: u64] - /// [generation: u64] [checksum: 32B] [compression: u8] + /// [generation: u64] [checksum: 32B] [compression: u8] [chunk_count: u32] /// -- free regions -- /// per region: /// [offset: u64] [size: u64] @@ -414,9 +425,9 @@ impl SegmentIndex { let entry_count = read_u32(data, &mut off)? as usize; let free_count = read_u32(data, &mut off)? as usize; - // Sanity-cap: the smallest possible entry is ~63 bytes (1-byte name), + // Sanity-cap: smallest entry is 64 bytes (2+1+8+8+8+32+1+4, 1-byte name), // free region is 16 bytes. Reject clearly corrupted counts early. - let max_entries = data.len() / 63; + let max_entries = data.len() / 64; let max_free = data.len() / 16; if entry_count > max_entries { return Err(CryptoError::VaultCorrupted(format!( @@ -454,6 +465,24 @@ impl SegmentIndex { let comp_byte = read_bytes(data, &mut off, 1)?; let compression = CompressionAlgorithm::from_u8(comp_byte[0])?; let chunk_count = read_u32(data, &mut off)?; + + // Validate chunk_count consistency with stored size + if chunk_count > 0 { + let expected = (chunk_count as u64) + .checked_mul(crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64) + .ok_or_else(|| { + CryptoError::VaultCorrupted(format!( + "segment '{name}': chunk_count overflows size" + )) + })?; + if size != expected { + return Err(CryptoError::VaultCorrupted(format!( + "segment '{name}': chunk_count {chunk_count} implies size \ + {expected} but stored {size}" + ))); + } + } + entries.push(SegmentEntry { name, offset, @@ -1569,18 +1598,55 @@ mod tests { assert!(parsed.entries[1].is_streaming()); } + #[test] + fn test_chunk_count_size_mismatch_rejected() { + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + + let mut idx = SegmentIndex::new(1024 * 1024); + // chunk_count=5 but size is wrong (should be 5 * ENCRYPTED_CHUNK_SIZE) + idx.entries.push(SegmentEntry { + name: "bad.bin".to_string(), + offset: 0, + size: 1024, // mismatched + generation: 0, + checksum: dummy_checksum(0), + compression: CompressionAlgorithm::None, + chunk_count: 5, + }); + + let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let result = SegmentIndex::from_bytes(&bytes); + assert!(result.is_err()); + let err = result.expect_err("should fail").to_string(); + assert!(err.contains("chunk_count"), "Error: {err}"); + + // Verify correct size is accepted + let mut idx2 = SegmentIndex::new(1024 * 1024); + idx2.entries.push(SegmentEntry { + name: "ok.bin".to_string(), + offset: 0, + size: 5 * ENCRYPTED_CHUNK_SIZE as u64, + generation: 0, + checksum: dummy_checksum(0), + compression: CompressionAlgorithm::None, + chunk_count: 5, + }); + let bytes2 = idx2.to_bytes(compute_index_size(idx2.capacity)).expect("serialize"); + assert!(SegmentIndex::from_bytes(&bytes2).is_ok()); + } + #[test] fn test_streaming_segment_size_zero() { - assert_eq!(streaming_segment_size(0), 0); + assert_eq!(streaming_segment_size(0).unwrap(), 0); } #[test] fn test_streaming_segment_size_single_chunk() { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; // 1 byte → 1 chunk - assert_eq!(streaming_segment_size(1), ENCRYPTED_CHUNK_SIZE as u64); + assert_eq!(streaming_segment_size(1).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); // Exactly one chunk - assert_eq!(streaming_segment_size(64 * 1024), ENCRYPTED_CHUNK_SIZE as u64); + assert_eq!(streaming_segment_size(64 * 1024).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); } #[test] @@ -1588,23 +1654,33 @@ mod tests { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; // 64KB + 1 byte → 2 chunks assert_eq!( - streaming_segment_size(64 * 1024 + 1), + streaming_segment_size(64 * 1024 + 1).unwrap(), 2 * ENCRYPTED_CHUNK_SIZE as u64 ); // 5 full chunks assert_eq!( - streaming_segment_size(5 * 64 * 1024), + streaming_segment_size(5 * 64 * 1024).unwrap(), 5 * ENCRYPTED_CHUNK_SIZE as u64 ); } + #[test] + fn test_streaming_segment_size_overflow() { + assert!(streaming_segment_size(u64::MAX).is_err()); + } + #[test] fn test_streaming_chunk_count_values() { - assert_eq!(streaming_chunk_count(0), 0); - assert_eq!(streaming_chunk_count(1), 1); - assert_eq!(streaming_chunk_count(64 * 1024), 1); - assert_eq!(streaming_chunk_count(64 * 1024 + 1), 2); - assert_eq!(streaming_chunk_count(5 * 64 * 1024), 5); + assert_eq!(streaming_chunk_count(0).unwrap(), 0); + assert_eq!(streaming_chunk_count(1).unwrap(), 1); + assert_eq!(streaming_chunk_count(64 * 1024).unwrap(), 1); + assert_eq!(streaming_chunk_count(64 * 1024 + 1).unwrap(), 2); + assert_eq!(streaming_chunk_count(5 * 64 * 1024).unwrap(), 5); + } + + #[test] + fn test_streaming_chunk_count_overflow() { + assert!(streaming_chunk_count(u64::MAX).is_err()); } #[test] diff --git a/rust/src/core/evfs/segment.rs b/rust/src/core/evfs/segment.rs index ae05d07..4480f90 100644 --- a/rust/src/core/evfs/segment.rs +++ b/rust/src/core/evfs/segment.rs @@ -104,20 +104,54 @@ pub fn derive_segment_nonce( // --------------------------------------------------------------------------- /// Per-chunk AAD for vault streaming segments. -/// Wire-compatible with the standalone streaming format's `ChunkAad`. -pub type VaultChunkAad = crate::core::streaming::ChunkAad; +/// +/// Extends standalone `ChunkAad` with `generation` to bind each chunk to its +/// specific segment write, preventing cross-segment splice attacks. +/// +/// Wire format: `[generation: u64 LE] [chunk_index: u64 LE] [is_final: u8]` = 17 bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VaultChunkAad { + pub generation: u64, + pub chunk_index: u64, + pub is_final: bool, +} + +/// Wire size of `VaultChunkAad`. +pub const VAULT_CHUNK_AAD_SIZE: usize = 17; + +impl VaultChunkAad { + pub fn to_bytes(self) -> [u8; VAULT_CHUNK_AAD_SIZE] { + let mut buf = [0u8; VAULT_CHUNK_AAD_SIZE]; + buf[0..8].copy_from_slice(&self.generation.to_le_bytes()); + buf[8..16].copy_from_slice(&self.chunk_index.to_le_bytes()); + buf[16] = u8::from(self.is_final); + buf + } +} /// Derive a unique nonce for a specific chunk within a streaming segment. /// -/// Reuses the existing HKDF-based nonce derivation with `chunk_index` as the -/// index parameter and the segment's `generation` counter, producing unique -/// nonces per chunk and per overwrite. +/// Uses a domain-separated HKDF info (`0x01 || chunk_index || generation`) +/// to ensure chunk nonces never collide with monolithic segment nonces +/// (which use `segment_index || generation` without a domain prefix). pub fn derive_chunk_nonce( nonce_key: &[u8], chunk_index: u64, generation: u64, ) -> Result, CryptoError> { - derive_segment_nonce(nonce_key, chunk_index, generation, NONCE_LEN) + let hk = Hkdf::::from_prk(nonce_key) + .map_err(|_| CryptoError::KdfFailed("nonce_key too short for HKDF-PRK".into()))?; + + // 17-byte info: domain(1) || chunk_index(LE8) || generation(LE8) + let mut info = [0u8; 17]; + info[0] = 0x01; + info[1..9].copy_from_slice(&chunk_index.to_le_bytes()); + info[9..17].copy_from_slice(&generation.to_le_bytes()); + + let mut nonce = vec![0u8; NONCE_LEN]; + hk.expand(&info, &mut nonce) + .map_err(|_| CryptoError::KdfFailed("HKDF expand failed for chunk nonce".into()))?; + Ok(nonce) } // --------------------------------------------------------------------------- @@ -961,29 +995,43 @@ mod tests { } #[test] - fn test_chunk_nonce_matches_segment_nonce() { - // derive_chunk_nonce(key, chunk_idx, gen) must equal - // derive_segment_nonce(key, chunk_idx, gen, NONCE_LEN) + fn test_chunk_nonce_domain_separation() { + // derive_chunk_nonce uses a domain-separated HKDF info, so it must + // NOT produce the same nonce as derive_segment_nonce with identical + // (index, generation) params. This prevents nonce collisions between + // monolithic segments and streaming chunk nonces. let keys = derive_vault_keys(&test_master_key()).expect("derive"); let chunk = derive_chunk_nonce(keys.nonce_key.as_bytes(), 42, 7).expect("chunk"); let segment = derive_segment_nonce(keys.nonce_key.as_bytes(), 42, 7, NONCE_LEN).expect("segment"); - assert_eq!(chunk, segment); + assert_ne!(chunk, segment); } #[test] - fn test_vault_chunk_aad_wire_compat() { - // VaultChunkAad is a type alias for ChunkAad — verify wire format - use crate::core::streaming::AAD_SIZE; + fn test_vault_chunk_aad_wire_format() { let aad = VaultChunkAad { - index: 99, + generation: 3, + chunk_index: 99, is_final: true, }; let bytes = aad.to_bytes(); - assert_eq!(bytes.len(), AAD_SIZE); - // index = 99 LE - assert_eq!(u64::from_le_bytes(bytes[0..8].try_into().unwrap()), 99); + assert_eq!(bytes.len(), VAULT_CHUNK_AAD_SIZE); + // generation = 3 LE + assert_eq!(u64::from_le_bytes(bytes[0..8].try_into().unwrap()), 3); + // chunk_index = 99 LE + assert_eq!(u64::from_le_bytes(bytes[8..16].try_into().unwrap()), 99); // is_final = true - assert_eq!(bytes[8], 1); + assert_eq!(bytes[16], 1); + } + + #[test] + fn test_vault_chunk_aad_not_final() { + let aad = VaultChunkAad { + generation: 0, + chunk_index: 0, + is_final: false, + }; + let bytes = aad.to_bytes(); + assert_eq!(bytes[16], 0); } } diff --git a/rust/src/core/streaming.rs b/rust/src/core/streaming.rs index 5b60766..c8f5359 100644 --- a/rust/src/core/streaming.rs +++ b/rust/src/core/streaming.rs @@ -173,12 +173,20 @@ impl ChunkAad { buf } - pub fn from_bytes(bytes: &[u8; AAD_SIZE]) -> Self { + pub fn from_bytes(bytes: &[u8; AAD_SIZE]) -> Result { let index = u64::from_le_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]); - let is_final = bytes[8] != 0; - Self { index, is_final } + let is_final = match bytes[8] { + 0 => false, + 1 => true, + v => { + return Err(CryptoError::InvalidParameter(format!( + "non-canonical is_final byte: 0x{v:02X}" + ))) + } + }; + Ok(Self { index, is_final }) } } @@ -249,16 +257,18 @@ pub fn strip_last_chunk_padding(padded: &[u8]) -> Result, CryptoError> { let max_payload = CHUNK_SIZE - PADDING_PREFIX_SIZE; if real_len > max_payload { - return Err(CryptoError::InvalidParameter(format!( - "Invalid padding length: {real_len}, max {max_payload}" - ))); + // NOTE: both error paths use the same message to prevent padding oracles. + // Padding is only stripped after AEAD verification, but defense in depth. + return Err(CryptoError::InvalidParameter( + "malformed chunk padding".to_string(), + )); } // Reject non-zero bytes in padding region let padding_start = PADDING_PREFIX_SIZE + real_len; if !padded[padding_start..].iter().all(|&b| b == 0) { return Err(CryptoError::InvalidParameter( - "Non-zero bytes in padding region".to_string(), + "malformed chunk padding".to_string(), )); } @@ -562,7 +572,7 @@ mod tests { // is_final = true assert_eq!(bytes[8], 1); - let roundtrip = ChunkAad::from_bytes(&bytes); + let roundtrip = ChunkAad::from_bytes(&bytes).expect("parse"); assert_eq!(roundtrip, aad); // Also test is_final = false @@ -572,7 +582,7 @@ mod tests { }; let bytes2 = aad2.to_bytes(); assert_eq!(bytes2[8], 0); - assert_eq!(ChunkAad::from_bytes(&bytes2), aad2); + assert_eq!(ChunkAad::from_bytes(&bytes2).expect("parse"), aad2); } #[test] @@ -738,6 +748,13 @@ mod tests { ); } + #[test] + fn test_aad_non_canonical_is_final_rejected() { + let mut bytes = [0u8; AAD_SIZE]; + bytes[8] = 0x02; // non-canonical + assert!(ChunkAad::from_bytes(&bytes).is_err()); + } + #[test] fn test_algorithm_conversion_roundtrip() { let algo: Algorithm = StreamAlgorithm::AesGcm.into(); From 27c912da5c19ed3226e4258f9ce5c67073afa1a0 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Sat, 21 Mar 2026 16:07:14 +0100 Subject: [PATCH 03/21] fix(evfs): account for final padded chunk in streaming size calculations streaming_chunk_count and streaming_segment_size did not account for the length-prefix padding on the final chunk (max payload CHUNK_SIZE-4). CHUNK_SIZE-aligned data now gets an extra empty padded chunk, and empty segments produce 1 padded chunk. Matches standalone streaming encrypt. --- rust/src/core/evfs/format.rs | 57 +++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/rust/src/core/evfs/format.rs b/rust/src/core/evfs/format.rs index 3ba90a5..975ae67 100644 --- a/rust/src/core/evfs/format.rs +++ b/rust/src/core/evfs/format.rs @@ -79,16 +79,16 @@ pub fn total_vault_size(capacity: u64, index_pad_size: usize) -> Result Result { - use crate::core::streaming::{CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE}; - if plaintext_size == 0 { - return Ok(0); - } - let chunk_count = plaintext_size.div_ceil(CHUNK_SIZE as u64); - chunk_count + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + let count = streaming_chunk_count(plaintext_size)? as u64; + count .checked_mul(ENCRYPTED_CHUNK_SIZE as u64) .ok_or_else(|| { CryptoError::InvalidParameter("streaming segment size overflows u64".into()) @@ -97,13 +97,23 @@ pub fn streaming_segment_size(plaintext_size: u64) -> Result { /// Compute the number of chunks needed for a given plaintext size. /// +/// The final chunk always uses `pad_last_chunk` (4-byte length prefix), +/// so its max payload is `CHUNK_SIZE - 4`. When the plaintext is exactly +/// CHUNK_SIZE-aligned, an extra empty padded chunk is appended. An empty +/// segment (0 bytes) produces one chunk containing padded empty data. +/// /// Returns `Err` if the count exceeds `u32::MAX`. pub fn streaming_chunk_count(plaintext_size: u64) -> Result { use crate::core::streaming::CHUNK_SIZE; if plaintext_size == 0 { - return Ok(0); - } - let count = plaintext_size.div_ceil(CHUNK_SIZE as u64); + return Ok(1); + } + let base = plaintext_size.div_ceil(CHUNK_SIZE as u64); + let count = if plaintext_size.is_multiple_of(CHUNK_SIZE as u64) { + base + 1 + } else { + base + }; u32::try_from(count).map_err(|_| { CryptoError::InvalidParameter("streaming chunk count exceeds u32::MAX".into()) }) @@ -1637,30 +1647,32 @@ mod tests { #[test] fn test_streaming_segment_size_zero() { - assert_eq!(streaming_segment_size(0).unwrap(), 0); + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + // 0 bytes → 1 padded empty chunk + assert_eq!(streaming_segment_size(0).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); } #[test] fn test_streaming_segment_size_single_chunk() { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; - // 1 byte → 1 chunk + // 1 byte → 1 padded chunk assert_eq!(streaming_segment_size(1).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); - // Exactly one chunk - assert_eq!(streaming_segment_size(64 * 1024).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); + // Exactly CHUNK_SIZE → 1 full + 1 empty padded = 2 chunks + assert_eq!(streaming_segment_size(64 * 1024).unwrap(), 2 * ENCRYPTED_CHUNK_SIZE as u64); } #[test] fn test_streaming_segment_size_multiple_chunks() { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; - // 64KB + 1 byte → 2 chunks + // 64KB + 1 byte → 2 chunks (last is partial, padded) assert_eq!( streaming_segment_size(64 * 1024 + 1).unwrap(), 2 * ENCRYPTED_CHUNK_SIZE as u64 ); - // 5 full chunks + // 5 * 64KB → 5 full + 1 empty padded = 6 chunks assert_eq!( streaming_segment_size(5 * 64 * 1024).unwrap(), - 5 * ENCRYPTED_CHUNK_SIZE as u64 + 6 * ENCRYPTED_CHUNK_SIZE as u64 ); } @@ -1671,11 +1683,16 @@ mod tests { #[test] fn test_streaming_chunk_count_values() { - assert_eq!(streaming_chunk_count(0).unwrap(), 0); + // 0 bytes → 1 empty padded chunk + assert_eq!(streaming_chunk_count(0).unwrap(), 1); + // 1 byte → 1 padded chunk assert_eq!(streaming_chunk_count(1).unwrap(), 1); - assert_eq!(streaming_chunk_count(64 * 1024).unwrap(), 1); + // Exactly CHUNK_SIZE → 1 full + 1 empty padded = 2 + assert_eq!(streaming_chunk_count(64 * 1024).unwrap(), 2); + // CHUNK_SIZE + 1 → 2 chunks (last is partial, padded) assert_eq!(streaming_chunk_count(64 * 1024 + 1).unwrap(), 2); - assert_eq!(streaming_chunk_count(5 * 64 * 1024).unwrap(), 5); + // 5 * CHUNK_SIZE → 5 full + 1 empty padded = 6 + assert_eq!(streaming_chunk_count(5 * 64 * 1024).unwrap(), 6); } #[test] From ffb254ff854e2674b903ac60c7717ab389f2a56e Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Sat, 21 Mar 2026 16:08:23 +0100 Subject: [PATCH 04/21] feat(evfs): add chunk encrypt/decrypt helpers for vault streaming aead_encrypt_with_key: exposes raw AEAD encrypt with caller-supplied nonce for streaming writes where nonces are derived deterministically. decrypt_vault_chunk: decrypts a single vault streaming chunk with constant-time nonce verification and VaultChunkAad authentication. --- rust/src/core/evfs/segment.rs | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/rust/src/core/evfs/segment.rs b/rust/src/core/evfs/segment.rs index 4480f90..214eef5 100644 --- a/rust/src/core/evfs/segment.rs +++ b/rust/src/core/evfs/segment.rs @@ -277,6 +277,20 @@ fn aead_decrypt_chacha( // AEAD helpers for vault API (random nonce, stored nonce) // --------------------------------------------------------------------------- +/// Encrypt with a caller-supplied nonce. Returns `ciphertext || tag`. +/// +/// Used by vault streaming writes where the nonce is derived deterministically +/// and prepended by the caller. +pub fn aead_encrypt_with_key( + key: &[u8], + nonce: &[u8], + plaintext: &[u8], + aad: &[u8], + algorithm: Algorithm, +) -> Result, CryptoError> { + aead_encrypt(key, nonce, plaintext, aad, algorithm) +} + /// Encrypt with a random nonce. Returns `nonce || ciphertext || tag`. pub fn aead_encrypt_random_nonce( key: &[u8], @@ -294,6 +308,41 @@ pub fn aead_encrypt_random_nonce( Ok(output) } +/// Decrypt a single vault streaming chunk. +/// +/// Input is `nonce || ciphertext || tag`. The stored nonce is verified +/// against the derived chunk nonce using constant-time comparison. +pub fn decrypt_vault_chunk( + cipher_key: &[u8], + nonce_key: &[u8], + algorithm: Algorithm, + chunk_data: &[u8], + chunk_index: u64, + generation: u64, + is_final: bool, +) -> Result, CryptoError> { + if chunk_data.len() < NONCE_LEN + TAG_LEN { + return Err(CryptoError::AuthenticationFailed); + } + + let (stored_nonce, ct_tag) = chunk_data.split_at(NONCE_LEN); + + // Verify nonce matches derived value + let expected_nonce = derive_chunk_nonce(nonce_key, chunk_index, generation)?; + if stored_nonce.ct_ne(&expected_nonce).into() { + return Err(CryptoError::AuthenticationFailed); + } + + let aad = VaultChunkAad { + generation, + chunk_index, + is_final, + } + .to_bytes(); + + aead_decrypt(cipher_key, stored_nonce, ct_tag, &aad, algorithm) +} + /// Decrypt data where the nonce is stored as a prefix. /// Input: `nonce || ciphertext || tag`. pub fn aead_decrypt_with_stored_nonce( From 44fac821d3bf893df53243b1080e09f0e80e9370 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Sat, 21 Mar 2026 16:08:59 +0100 Subject: [PATCH 05/21] feat(evfs): implement vault_write_stream with constant-memory chunking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Streaming write processes data in 64KB encrypted chunks — peak memory stays at ~64KB regardless of segment size. Caller provides total size upfront for pre-allocation. - Buffers arbitrary-sized input into CHUNK_SIZE pieces - Incremental BLAKE3 checksum over original plaintext - Per-chunk nonce derivation, AEAD encrypt, fsync for crash safety - WAL-protected: rollback restores previous index on crash - Extends vault_read to handle streaming segments (chunk_count > 0) --- rust/src/api/evfs/mod.rs | 280 ++++++++++++++++++++++++++++++++++++- rust/src/api/evfs/tests.rs | 278 ++++++++++++++++++++++++++++++++++++ 2 files changed, 556 insertions(+), 2 deletions(-) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index d5e8d51..3103e8b 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -307,7 +307,8 @@ pub fn vault_write( Ok(()) } -/// Read a named segment. Decompression is automatic. +/// Read a named segment. Handles both monolithic and streaming segments. +/// Decompression is automatic for monolithic segments. #[cfg(feature = "compression")] pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, CryptoError> { let entry = handle @@ -320,8 +321,20 @@ pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, Cry let seg_gen = entry.generation; let seg_compression = entry.compression; let seg_checksum = entry.checksum; + let seg_chunk_count = entry.chunk_count; + + if seg_chunk_count > 0 { + return vault_read_streaming( + handle, + &name, + seg_offset, + seg_gen, + seg_checksum, + seg_chunk_count, + ); + } - // Read encrypted data from disk + // Monolithic segment — read all at once let read_len = usize::try_from(seg_size).map_err(|_| { CryptoError::VaultCorrupted(format!( "segment size {seg_size} exceeds platform address space" @@ -353,6 +366,269 @@ pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, Cry Ok(plaintext) } +/// Read a streaming (chunked) segment, decrypting each chunk individually. +#[cfg(feature = "compression")] +fn vault_read_streaming( + handle: &mut VaultHandle, + name: &str, + seg_offset: u64, + generation: u64, + expected_checksum: [u8; 32], + chunk_count: u32, +) -> Result, CryptoError> { + use crate::core::streaming::{strip_last_chunk_padding, ENCRYPTED_CHUNK_SIZE}; + + let data_off = format::data_region_offset(handle.index_pad_size); + let cipher_key = handle.keys.cipher_key.as_bytes(); + let nonce_key = handle.keys.nonce_key.as_bytes(); + let algorithm = handle.algorithm; + let mut plaintext = Vec::new(); + let mut chunk_buf = vec![0u8; ENCRYPTED_CHUNK_SIZE]; + + for i in 0..chunk_count as u64 { + let chunk_offset = data_off + seg_offset + i * ENCRYPTED_CHUNK_SIZE as u64; + handle.file.seek(SeekFrom::Start(chunk_offset))?; + handle.file.read_exact(&mut chunk_buf)?; + + let is_final = i == (chunk_count as u64 - 1); + let decrypted = segment::decrypt_vault_chunk( + cipher_key, + nonce_key, + algorithm, + &chunk_buf, + i, + generation, + is_final, + )?; + + if is_final { + let real_data = strip_last_chunk_padding(&decrypted)?; + plaintext.extend_from_slice(&real_data); + } else { + plaintext.extend_from_slice(&decrypted); + } + } + + // Verify checksum on full reassembled plaintext + if !segment::verify_checksum(&plaintext, &expected_checksum) { + return Err(CryptoError::VaultCorrupted(format!( + "integrity check failed for streaming segment '{name}'" + ))); + } + + Ok(plaintext) +} + +/// Write (or overwrite) a named segment using streaming chunked encryption. +/// +/// Data is processed in 64KB chunks — peak memory stays constant regardless +/// of segment size. The caller provides `total_plaintext_size` upfront so +/// that space can be pre-allocated in the vault. +/// +/// `data_stream` must yield exactly `total_plaintext_size` bytes total. +/// Chunks from the iterator can be any size; they are internally buffered +/// into CHUNK_SIZE pieces for encryption. +/// +/// Each encrypted chunk is fsynced individually for crash safety. On crash +/// mid-stream, WAL rollback restores the previous index and the partial +/// data is invisible (CSPRNG noise in pre-allocated space). +#[cfg(feature = "compression")] +pub fn vault_write_stream( + handle: &mut VaultHandle, + name: String, + total_plaintext_size: u64, + data_stream: impl Iterator>, +) -> Result<(), CryptoError> { + use crate::core::streaming::{pad_last_chunk, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE}; + + let expected_chunks = format::streaming_chunk_count(total_plaintext_size)?; + let total_encrypted_size = format::streaming_segment_size(total_plaintext_size)?; + + let gen = handle.index.next_gen(); + + // WAL journal old index before any mutation + let old_encrypted_index = read_encrypted_index( + &mut handle.file, + PRIMARY_INDEX_OFFSET, + format::encrypted_index_size(handle.index_pad_size), + )?; + handle + .wal + .begin(WalOp::WriteSegment, &old_encrypted_index)?; + + // If overwrite: secure-erase old region, deallocate + if let Some(old_entry) = handle.index.remove(&name) { + segment::secure_erase_region( + &mut handle.file, + format::data_region_offset(handle.index_pad_size) + old_entry.offset, + old_entry.size, + )?; + handle.index.deallocate(old_entry.offset, old_entry.size); + } + + // Allocate space for the entire streaming segment + let offset = handle.index.allocate(total_encrypted_size)?; + let data_off = format::data_region_offset(handle.index_pad_size); + + // Stream chunks: buffer input → CHUNK_SIZE pieces → encrypt → write + let mut hasher = blake3::Hasher::new(); + let mut chunk_buf = vec![0u8; CHUNK_SIZE]; + let mut buf_len = 0usize; + let mut total_received: u64 = 0; + let mut chunk_index: u64 = 0; + let nonce_key = handle.keys.nonce_key.as_bytes(); + let cipher_key = handle.keys.cipher_key.as_bytes(); + let algorithm = handle.algorithm; + + for mut input in data_stream { + hasher.update(&input); + total_received += input.len() as u64; + + if total_received > total_plaintext_size { + input.zeroize(); + chunk_buf.zeroize(); + return Err(CryptoError::InvalidParameter(format!( + "stream exceeded total_plaintext_size: received >{total_received}, \ + expected {total_plaintext_size}" + ))); + } + + let mut pos = 0; + while pos < input.len() { + let take = std::cmp::min(CHUNK_SIZE - buf_len, input.len() - pos); + chunk_buf[buf_len..buf_len + take].copy_from_slice(&input[pos..pos + take]); + buf_len += take; + pos += take; + + // Flush full buffers as non-final. The final padded chunk is + // always written after the loop (matching standalone encrypt). + if buf_len == CHUNK_SIZE { + write_encrypted_chunk( + &mut handle.file, + cipher_key, + nonce_key, + algorithm, + &chunk_buf[..CHUNK_SIZE], + chunk_index, + gen, + false, + data_off + offset + chunk_index * ENCRYPTED_CHUNK_SIZE as u64, + )?; + chunk_index += 1; + buf_len = 0; + } + } + input.zeroize(); + } + + // Validate total bytes received + if total_received != total_plaintext_size { + chunk_buf.zeroize(); + return Err(CryptoError::InvalidParameter(format!( + "stream underflow: received {total_received} bytes, \ + expected {total_plaintext_size}" + ))); + } + + // Write final padded chunk (may be empty for CHUNK_SIZE-aligned data) + let padded = pad_last_chunk(&chunk_buf[..buf_len])?; + chunk_buf.zeroize(); + write_encrypted_chunk( + &mut handle.file, + cipher_key, + nonce_key, + algorithm, + &padded, + chunk_index, + gen, + true, + data_off + offset + chunk_index * ENCRYPTED_CHUNK_SIZE as u64, + )?; + + // Verify chunk count matches expectation + let actual_chunks = chunk_index + 1; + if actual_chunks != expected_chunks as u64 { + return Err(CryptoError::VaultCorrupted(format!( + "chunk count mismatch: wrote {actual_chunks}, expected {expected_chunks}" + ))); + } + + // Finalize checksum + let checksum: [u8; 32] = hasher.finalize().into(); + + // Update index + let entry = SegmentEntry::new( + &name, + offset, + total_encrypted_size, + gen, + checksum, + CompressionAlgorithm::None, + expected_chunks, + )?; + handle.index.add(entry)?; + + // Flush index (primary + shadow) + flush_index( + &mut handle.file, + &handle.index, + &handle.keys, + handle.algorithm, + handle.index.capacity, + handle.index_pad_size, + )?; + + // WAL commit + handle.wal.commit()?; + + Ok(()) +} + +/// Encrypt a single plaintext chunk and write it to the vault file at the given +/// absolute offset. Fsyncs after write for crash safety. +#[allow(clippy::too_many_arguments)] +fn write_encrypted_chunk( + file: &mut File, + cipher_key: &[u8], + nonce_key: &[u8], + algorithm: crate::core::format::Algorithm, + plaintext: &[u8], + chunk_index: u64, + generation: u64, + is_final: bool, + abs_offset: u64, +) -> Result<(), CryptoError> { + use crate::core::streaming::{CHUNK_SIZE, NONCE_SIZE}; + + if plaintext.len() != CHUNK_SIZE { + return Err(CryptoError::InvalidParameter(format!( + "chunk plaintext must be {CHUNK_SIZE} bytes, got {}", + plaintext.len() + ))); + } + + let nonce = segment::derive_chunk_nonce(nonce_key, chunk_index, generation)?; + let aad = segment::VaultChunkAad { + generation, + chunk_index, + is_final, + } + .to_bytes(); + + let ct_tag = segment::aead_encrypt_with_key(cipher_key, &nonce, plaintext, &aad, algorithm)?; + + // Wire format: nonce || ciphertext || tag + let mut wire = Vec::with_capacity(NONCE_SIZE + ct_tag.len()); + wire.extend_from_slice(&nonce); + wire.extend_from_slice(&ct_tag); + + file.seek(SeekFrom::Start(abs_offset))?; + file.write_all(&wire)?; + file.sync_all()?; + + Ok(()) +} + /// Delete a named segment. The region is secure-erased and returned to the /// free list for reuse by future writes. #[cfg(feature = "compression")] diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index 1d16cdf..00edfd1 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -1636,3 +1636,281 @@ fn test_vault_health_full_capacity() { assert_eq!(h.fragmentation_ratio, 0.0); assert_eq!(h.used_bytes + h.free_list_bytes + h.unallocated_bytes, h.total_bytes); } + +// -- Streaming Write -------------------------------------------------------- + +/// Helper: stream-write data in fixed-size pieces. +fn stream_write_chunks( + handle: &mut VaultHandle, + name: &str, + data: &[u8], + piece_size: usize, +) -> Result<(), CryptoError> { + let chunks: Vec> = data + .chunks(piece_size) + .map(|c| c.to_vec()) + .collect(); + vault_write_stream( + handle, + name.to_string(), + data.len() as u64, + chunks.into_iter(), + ) +} + +#[test] +fn test_stream_write_read_roundtrip() { + let dir = tempfile::tempdir().expect("tempdir"); + // 2MB vault to fit the streaming overhead + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data = vec![0x42u8; 200_000]; // ~3 chunks + stream_write_chunks(&mut handle, "video.bin", &data, 4096).expect("stream write"); + + let readback = vault_read(&mut handle, "video.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_single_byte() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let data = vec![0xAA; 1]; + stream_write_chunks(&mut handle, "tiny.bin", &data, 1).expect("stream write"); + + let readback = vault_read(&mut handle, "tiny.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_empty_segment() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + // 0 bytes — still produces 1 padded chunk + vault_write_stream( + &mut handle, + "empty.bin".into(), + 0, + std::iter::empty(), + ) + .expect("stream write empty"); + + let readback = vault_read(&mut handle, "empty.bin".into()).expect("read"); + assert!(readback.is_empty()); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_exact_chunk_boundary() { + use crate::core::streaming::CHUNK_SIZE; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + // Exactly CHUNK_SIZE bytes — triggers the extra empty padded chunk + let data = vec![0xBB; CHUNK_SIZE]; + stream_write_chunks(&mut handle, "aligned.bin", &data, CHUNK_SIZE).expect("stream write"); + + let readback = vault_read(&mut handle, "aligned.bin".into()).expect("read"); + assert_eq!(readback, data); + + // Verify chunk_count = 2 (1 full + 1 empty padded) + let entry = handle.index.find("aligned.bin").expect("entry"); + assert_eq!(entry.chunk_count, 2); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_overwrite_existing() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + // Write original (monolithic) + vault_write( + &mut handle, + "doc.txt".into(), + b"original data".to_vec(), + None, + ) + .expect("write"); + + // Overwrite with streaming (larger) + let new_data = vec![0xCC; 100_000]; + stream_write_chunks(&mut handle, "doc.txt", &new_data, 8192).expect("stream overwrite"); + + let readback = vault_read(&mut handle, "doc.txt".into()).expect("read"); + assert_eq!(readback, new_data); + + // Verify it's now a streaming segment + let entry = handle.index.find("doc.txt").expect("entry"); + assert!(entry.is_streaming()); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_overwrite_with_smaller() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + // Write original (streaming, large) + let original = vec![0xAA; 200_000]; + stream_write_chunks(&mut handle, "file.bin", &original, 4096).expect("write large"); + + // Overwrite with smaller streaming segment + let smaller = vec![0xBB; 1000]; + stream_write_chunks(&mut handle, "file.bin", &smaller, 500).expect("write small"); + + let readback = vault_read(&mut handle, "file.bin".into()).expect("read"); + assert_eq!(readback, smaller); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_wrong_size_too_few_bytes() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + // Claim 1000 bytes but provide only 500 + let data = vec![0xAA; 500]; + let result = vault_write_stream( + &mut handle, + "bad.bin".into(), + 1000, + vec![data].into_iter(), + ); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("underflow"), "Expected underflow error: {err}"); +} + +#[test] +fn test_stream_write_wrong_size_too_many_bytes() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + // Claim 500 bytes but provide 1000 + let data = vec![0xAA; 1000]; + let result = vault_write_stream( + &mut handle, + "bad.bin".into(), + 500, + vec![data].into_iter(), + ); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("exceeded"), "Expected exceeded error: {err}"); +} + +#[test] +fn test_stream_write_persist_reopen() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = vault_path(&dir); + + let data = vec![0xDD; 150_000]; + { + let mut handle = + vault_create(path.clone(), test_key(), "aes-256-gcm".into(), 2 * 1024 * 1024) + .expect("create"); + stream_write_chunks(&mut handle, "persist.bin", &data, 4096).expect("stream write"); + vault_close(handle).expect("close"); + } + + // Reopen and verify + let mut handle = vault_open(path, test_key()).expect("open"); + let readback = vault_read(&mut handle, "persist.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_chacha20() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir + .path() + .join("chacha.vault") + .to_str() + .expect("path") + .to_string(); + let mut handle = + vault_create(path, test_key(), "chacha20-poly1305".into(), 2 * 1024 * 1024) + .expect("create"); + + let data = vec![0xEE; 100_000]; + stream_write_chunks(&mut handle, "chacha.bin", &data, 8192).expect("stream write"); + + let readback = vault_read(&mut handle, "chacha.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_arbitrary_input_sizes() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + // Provide data in irregular chunks (1, 7, 100, 50000, 3, ...) + let total = 100_000usize; + let data: Vec = (0..total).map(|i| (i % 256) as u8).collect(); + + let pieces = vec![ + data[..1].to_vec(), + data[1..8].to_vec(), + data[8..108].to_vec(), + data[108..50108].to_vec(), + data[50108..50111].to_vec(), + data[50111..].to_vec(), + ]; + + vault_write_stream( + &mut handle, + "irregular.bin".into(), + total as u64, + pieces.into_iter(), + ) + .expect("stream write"); + + let readback = vault_read(&mut handle, "irregular.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_write_coexists_with_monolithic() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + // Write monolithic + vault_write(&mut handle, "mono.txt".into(), b"mono data".to_vec(), None) + .expect("write mono"); + + // Write streaming + let stream_data = vec![0xFF; 80_000]; + stream_write_chunks(&mut handle, "stream.bin", &stream_data, 4096) + .expect("stream write"); + + // Read both back + let mono = vault_read(&mut handle, "mono.txt".into()).expect("read mono"); + assert_eq!(mono, b"mono data"); + + let stream = vault_read(&mut handle, "stream.bin".into()).expect("read stream"); + assert_eq!(stream, stream_data); + + // Verify types + assert!(!handle.index.find("mono.txt").unwrap().is_streaming()); + assert!(handle.index.find("stream.bin").unwrap().is_streaming()); + + vault_close(handle).expect("close"); +} From 3df3e9a5c714b2d83ddad8591c8d48c557c40283 Mon Sep 17 00:00:00 2001 From: Adel HB Date: Sat, 21 Mar 2026 16:26:38 +0100 Subject: [PATCH 06/21] feat(evfs): add chunked streaming to vault_read with integrity check --- rust/src/api/evfs/mod.rs | 163 +++++++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 22 deletions(-) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index d5e8d51..fe21869 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -16,6 +16,7 @@ use crate::core::evfs::format::{ use crate::core::evfs::segment::{self, SegmentCryptoParams}; use crate::core::evfs::wal::{VaultLock, WalOp, WriteAheadLog}; use crate::core::format::Algorithm; +use subtle::ConstantTimeEq; use zeroize::Zeroize; use std::fs::{File, OpenOptions}; @@ -59,7 +60,14 @@ pub fn vault_create( // Create empty index and flush to primary + shadow let index = SegmentIndex::new(capacity_bytes); - flush_index(&mut file, &index, &keys, algo, capacity_bytes, index_pad_size)?; + flush_index( + &mut file, + &index, + &keys, + algo, + capacity_bytes, + index_pad_size, + )?; // Create fresh WAL (checkpoint to clear any stale data) let mut wal = WriteAheadLog::open(&path)?; @@ -122,7 +130,8 @@ pub fn vault_open(path: String, mut key: Vec) -> Result) -> Result) -> Result Result, Cry let seg_gen = entry.generation; let seg_compression = entry.compression; let seg_checksum = entry.checksum; + let chunk_count = entry.chunk_count; + + // INTEROP: If the segment is chunked, reassemble it into a single vector + if chunk_count > 0 { + let data_region = format::data_region_offset(handle.index_pad_size); + let mut hasher = blake3::Hasher::new(); + let mut decompressor = if seg_compression != CompressionAlgorithm::None { + Some(crate::core::compression::streaming::new_decompressor( + seg_compression, + )?) + } else { + None + }; + + let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); + let mut full_plaintext = Vec::new(); + + for i in 0..chunk_count { + let chunk_offset = data_region + + seg_offset + + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); + handle.file.seek(SeekFrom::Start(chunk_offset))?; + + let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; + handle.file.read_exact(&mut encrypted)?; + + // Verify independent chunk nonce + let expected_nonce = + segment::derive_chunk_nonce(handle.keys.nonce_key.as_bytes(), i as u64, seg_gen)?; + let (stored_nonce, _) = encrypted.split_at(crate::core::streaming::NONCE_SIZE); + if stored_nonce.ct_ne(&expected_nonce).into() { + return Err(CryptoError::AuthenticationFailed); + } + + let is_final = i == chunk_count - 1; + let aad = segment::VaultChunkAad { + generation: seg_gen, + chunk_index: i as u64, + is_final, + } + .to_bytes(); + + let decrypted = segment::aead_decrypt_with_stored_nonce( + handle.keys.cipher_key.as_bytes(), + &encrypted, + &aad, + handle.algorithm, + )?; + + // Strip padding purely from the final chunk + let plaintext = if is_final { + crate::core::streaming::strip_last_chunk_padding(&decrypted)? + } else { + decrypted + }; + + let final_data = if let Some(ref mut dec) = decompressor { + dec.decompress_chunk(&plaintext, &mut decomp_buf)?; + if is_final { + dec.finish(&mut decomp_buf)?; + } + let data = decomp_buf.clone(); + decomp_buf.clear(); + data + } else { + plaintext + }; + + hasher.update(&final_data); + full_plaintext.extend_from_slice(&final_data); + } + + // Verify full file integrity using BLAKE3 Checksum + let actual_checksum = hasher.finalize(); + if actual_checksum.as_bytes().ct_ne(&seg_checksum).into() { + return Err(CryptoError::VaultCorrupted(format!( + "integrity check failed for segment '{name}'" + ))); + } + + return Ok(full_plaintext); + } - // Read encrypted data from disk + // Existing monolithic read logic (used when chunk_count == 0) let read_len = usize::try_from(seg_size).map_err(|_| { CryptoError::VaultCorrupted(format!( "segment size {seg_size} exceeds platform address space" )) })?; - handle - .file - .seek(SeekFrom::Start(format::data_region_offset(handle.index_pad_size) + seg_offset))?; + handle.file.seek(SeekFrom::Start( + format::data_region_offset(handle.index_pad_size) + seg_offset, + ))?; let mut encrypted = vec![0u8; read_len]; handle.file.read_exact(&mut encrypted)?; @@ -358,7 +461,11 @@ pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, Cry #[cfg(feature = "compression")] pub fn vault_delete(handle: &mut VaultHandle, name: String) -> Result<(), CryptoError> { // WAL journal old index - let old_encrypted_index = read_encrypted_index(&mut handle.file, PRIMARY_INDEX_OFFSET, format::encrypted_index_size(handle.index_pad_size))?; + let old_encrypted_index = read_encrypted_index( + &mut handle.file, + PRIMARY_INDEX_OFFSET, + format::encrypted_index_size(handle.index_pad_size), + )?; handle .wal .begin(WalOp::DeleteSegment, &old_encrypted_index)?; @@ -417,7 +524,11 @@ fn vault_resize_grow_impl( old_capacity: u64, new_capacity: u64, ) -> Result<(), CryptoError> { - let old_encrypted_index = read_encrypted_index(&mut handle.file, PRIMARY_INDEX_OFFSET, format::encrypted_index_size(handle.index_pad_size))?; + let old_encrypted_index = read_encrypted_index( + &mut handle.file, + PRIMARY_INDEX_OFFSET, + format::encrypted_index_size(handle.index_pad_size), + )?; handle.wal.begin(WalOp::UpdateIndex, &old_encrypted_index)?; // Extend file and CSPRNG-fill new space (also overwrites old shadow position) @@ -466,7 +577,11 @@ fn vault_resize_shrink_impl( } // WAL begin — journal the current encrypted index for crash recovery. - let old_encrypted_index = read_encrypted_index(&mut handle.file, PRIMARY_INDEX_OFFSET, format::encrypted_index_size(handle.index_pad_size))?; + let old_encrypted_index = read_encrypted_index( + &mut handle.file, + PRIMARY_INDEX_OFFSET, + format::encrypted_index_size(handle.index_pad_size), + )?; handle.wal.begin(WalOp::UpdateIndex, &old_encrypted_index)?; // Update index metadata for the new capacity. @@ -571,9 +686,9 @@ pub fn vault_defragment(handle: &mut VaultHandle) -> Result Result Date: Sat, 21 Mar 2026 16:59:53 +0100 Subject: [PATCH 07/21] fix(evfs): harden streaming write with checked arithmetic and zeroization --- rust/src/api/evfs/mod.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index 3103e8b..eebcddd 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -386,12 +386,12 @@ fn vault_read_streaming( let mut chunk_buf = vec![0u8; ENCRYPTED_CHUNK_SIZE]; for i in 0..chunk_count as u64 { - let chunk_offset = data_off + seg_offset + i * ENCRYPTED_CHUNK_SIZE as u64; + let chunk_offset = chunk_abs_offset(data_off, seg_offset, i)?; handle.file.seek(SeekFrom::Start(chunk_offset))?; handle.file.read_exact(&mut chunk_buf)?; let is_final = i == (chunk_count as u64 - 1); - let decrypted = segment::decrypt_vault_chunk( + let mut decrypted = segment::decrypt_vault_chunk( cipher_key, nonce_key, algorithm, @@ -403,9 +403,11 @@ fn vault_read_streaming( if is_final { let real_data = strip_last_chunk_padding(&decrypted)?; + decrypted.zeroize(); plaintext.extend_from_slice(&real_data); } else { plaintext.extend_from_slice(&decrypted); + decrypted.zeroize(); } } @@ -439,7 +441,7 @@ pub fn vault_write_stream( total_plaintext_size: u64, data_stream: impl Iterator>, ) -> Result<(), CryptoError> { - use crate::core::streaming::{pad_last_chunk, CHUNK_SIZE, ENCRYPTED_CHUNK_SIZE}; + use crate::core::streaming::{pad_last_chunk, CHUNK_SIZE}; let expected_chunks = format::streaming_chunk_count(total_plaintext_size)?; let total_encrypted_size = format::streaming_segment_size(total_plaintext_size)?; @@ -503,6 +505,7 @@ pub fn vault_write_stream( // Flush full buffers as non-final. The final padded chunk is // always written after the loop (matching standalone encrypt). if buf_len == CHUNK_SIZE { + let abs_off = chunk_abs_offset(data_off, offset, chunk_index)?; write_encrypted_chunk( &mut handle.file, cipher_key, @@ -512,7 +515,7 @@ pub fn vault_write_stream( chunk_index, gen, false, - data_off + offset + chunk_index * ENCRYPTED_CHUNK_SIZE as u64, + abs_off, )?; chunk_index += 1; buf_len = 0; @@ -531,9 +534,10 @@ pub fn vault_write_stream( } // Write final padded chunk (may be empty for CHUNK_SIZE-aligned data) - let padded = pad_last_chunk(&chunk_buf[..buf_len])?; + let mut padded = pad_last_chunk(&chunk_buf[..buf_len])?; chunk_buf.zeroize(); - write_encrypted_chunk( + let final_off = chunk_abs_offset(data_off, offset, chunk_index)?; + let final_result = write_encrypted_chunk( &mut handle.file, cipher_key, nonce_key, @@ -542,8 +546,10 @@ pub fn vault_write_stream( chunk_index, gen, true, - data_off + offset + chunk_index * ENCRYPTED_CHUNK_SIZE as u64, - )?; + final_off, + ); + padded.zeroize(); + final_result?; // Verify chunk count matches expectation let actual_chunks = chunk_index + 1; @@ -584,6 +590,15 @@ pub fn vault_write_stream( Ok(()) } +/// Compute the absolute file offset for a chunk, using checked arithmetic. +fn chunk_abs_offset(data_off: u64, seg_offset: u64, chunk_index: u64) -> Result { + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + chunk_index + .checked_mul(ENCRYPTED_CHUNK_SIZE as u64) + .and_then(|co| data_off.checked_add(seg_offset)?.checked_add(co)) + .ok_or_else(|| CryptoError::InvalidParameter("chunk offset overflow".into())) +} + /// Encrypt a single plaintext chunk and write it to the vault file at the given /// absolute offset. Fsyncs after write for crash safety. #[allow(clippy::too_many_arguments)] From dc93ee41e3e1afb473b188bc788cd02d5f4b68b9 Mon Sep 17 00:00:00 2001 From: Adel HB Date: Sat, 21 Mar 2026 17:00:24 +0100 Subject: [PATCH 08/21] feat(evfs): add vault_read_stream streaming reader --- rust/src/api/evfs/mod.rs | 111 +++++++++++++++++++++++++++++++++++++ rust/src/api/evfs/tests.rs | 64 +++++++++++++++++---- 2 files changed, 163 insertions(+), 12 deletions(-) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index fe21869..8209360 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -16,6 +16,7 @@ use crate::core::evfs::format::{ use crate::core::evfs::segment::{self, SegmentCryptoParams}; use crate::core::evfs::wal::{VaultLock, WalOp, WriteAheadLog}; use crate::core::format::Algorithm; +use crate::frb_generated::StreamSink; use subtle::ConstantTimeEq; use zeroize::Zeroize; @@ -456,6 +457,116 @@ pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, Cry Ok(plaintext) } +/// Read a named segment sequentially through a stream. Decompression is automatic. +#[cfg(feature = "compression")] +pub fn vault_read_stream( + handle: &mut VaultHandle, + name: String, + verify_checksum: bool, + sink: StreamSink>, + on_progress: StreamSink, +) -> Result<(), CryptoError> { + let entry = handle + .index + .find(&name) + .ok_or_else(|| CryptoError::SegmentNotFound(name.clone()))?; + + let seg_offset = entry.offset; + let seg_gen = entry.generation; + let seg_compression = entry.compression; + let seg_checksum = entry.checksum; + let chunk_count = entry.chunk_count; + + // INTEROP: If chunk_count is 0, this is a monolithic segment. Handle with a one-shot read. + if chunk_count == 0 { + let plaintext = vault_read(handle, name)?; + let _ = sink.add(plaintext); + let _ = on_progress.add(1.0); + return Ok(()); + } + + let data_region = format::data_region_offset(handle.index_pad_size); + let mut hasher = blake3::Hasher::new(); + let mut decompressor = if seg_compression != CompressionAlgorithm::None { + Some(crate::core::compression::streaming::new_decompressor( + seg_compression, + )?) + } else { + None + }; + + let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); + + for i in 0..chunk_count { + let chunk_offset = data_region + + seg_offset + + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); + handle.file.seek(SeekFrom::Start(chunk_offset))?; + + let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; + handle.file.read_exact(&mut encrypted)?; + + // Derive nonce exclusively for this chunk iteration + let expected_nonce = + segment::derive_chunk_nonce(handle.keys.nonce_key.as_bytes(), i as u64, seg_gen)?; + + let (stored_nonce, _) = encrypted.split_at(crate::core::streaming::NONCE_SIZE); + if stored_nonce.ct_ne(&expected_nonce).into() { + return Err(CryptoError::AuthenticationFailed); + } + + let is_final = i == chunk_count - 1; + let aad = segment::VaultChunkAad { + generation: seg_gen, + chunk_index: i as u64, + is_final, + } + .to_bytes(); + + let decrypted = segment::aead_decrypt_with_stored_nonce( + handle.keys.cipher_key.as_bytes(), + &encrypted, + &aad, + handle.algorithm, + )?; + + let plaintext = if is_final { + crate::core::streaming::strip_last_chunk_padding(&decrypted)? + } else { + decrypted + }; + + let final_data = if let Some(ref mut dec) = decompressor { + dec.decompress_chunk(&plaintext, &mut decomp_buf)?; + if is_final { + dec.finish(&mut decomp_buf)?; + } + let data = decomp_buf.clone(); + decomp_buf.clear(); + data + } else { + plaintext + }; + + hasher.update(&final_data); + + // Pass data back to Flutter side ignoring results on dropped listener endpoints + let _ = sink.add(final_data); + let _ = on_progress.add(((i + 1) as f64) / (chunk_count as f64)); + } + + if verify_checksum { + let actual_checksum = hasher.finalize(); + if actual_checksum.as_bytes().ct_ne(&seg_checksum).into() { + return Err(CryptoError::VaultCorrupted(format!( + "integrity check failed for segment '{name}'" + ))); + } + } + + Ok(()) +} + /// Delete a named segment. The region is secure-erased and returned to the /// free list for reuse by future writes. #[cfg(feature = "compression")] diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index 1d16cdf..adb3faf 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -1,4 +1,3 @@ - use super::*; use crate::core::evfs::format::{encrypted_index_size, MIN_INDEX_PAD_SIZE}; @@ -82,7 +81,12 @@ fn test_open_runs_wal_recovery() { // Save the "good" encrypted index (containing only A) let good_encrypted = { let mut f = File::open(&path).expect("open"); - read_encrypted_index(&mut f, PRIMARY_INDEX_OFFSET, encrypted_index_size(MIN_INDEX_PAD_SIZE)).expect("read index") + read_encrypted_index( + &mut f, + PRIMARY_INDEX_OFFSET, + encrypted_index_size(MIN_INDEX_PAD_SIZE), + ) + .expect("read index") }; // Add segment B normally (both index and data on disk) @@ -876,7 +880,12 @@ fn test_resize_grow_crash_recovery() { // Save the "good" encrypted index (1MB capacity, contains A). let good_encrypted = { let mut f = File::open(&path).expect("open"); - read_encrypted_index(&mut f, PRIMARY_INDEX_OFFSET, encrypted_index_size(MIN_INDEX_PAD_SIZE)).expect("read index") + read_encrypted_index( + &mut f, + PRIMARY_INDEX_OFFSET, + encrypted_index_size(MIN_INDEX_PAD_SIZE), + ) + .expect("read index") }; // Perform a real grow to 2MB. @@ -989,7 +998,12 @@ fn test_resize_grow_crash_midway_recovers() { // Capture the good encrypted index before simulating a partial grow. { let mut f = File::open(&path).expect("open"); - good_encrypted = read_encrypted_index(&mut f, PRIMARY_INDEX_OFFSET, encrypted_index_size(MIN_INDEX_PAD_SIZE)).expect("read idx"); + good_encrypted = read_encrypted_index( + &mut f, + PRIMARY_INDEX_OFFSET, + encrypted_index_size(MIN_INDEX_PAD_SIZE), + ) + .expect("read idx"); } // Simulate a crash mid-grow: extend the file and CSPRNG-fill @@ -1003,7 +1017,12 @@ fn test_resize_grow_crash_midway_recovers() { let new_total = format::total_vault_size(2 * SIZE_MB, MIN_INDEX_PAD_SIZE).expect("size"); f.set_len(new_total).expect("extend"); // CSPRNG-fill overwrites old shadow position at format::data_region_offset(MIN_INDEX_PAD_SIZE) + SIZE_MB - segment::secure_erase_region(&mut f, format::data_region_offset(MIN_INDEX_PAD_SIZE) + SIZE_MB, SIZE_MB).expect("fill"); + segment::secure_erase_region( + &mut f, + format::data_region_offset(MIN_INDEX_PAD_SIZE) + SIZE_MB, + SIZE_MB, + ) + .expect("fill"); let mut wal = WriteAheadLog::open(&path).expect("wal"); wal.begin(WalOp::UpdateIndex, &good_encrypted) @@ -1210,7 +1229,12 @@ fn test_defragment_crash_recovery() { // Save pre-defrag encrypted index (the "good" state) let pre_defrag_index = { let mut f = File::open(&path).expect("open"); - read_encrypted_index(&mut f, PRIMARY_INDEX_OFFSET, encrypted_index_size(MIN_INDEX_PAD_SIZE)).expect("read index") + read_encrypted_index( + &mut f, + PRIMARY_INDEX_OFFSET, + encrypted_index_size(MIN_INDEX_PAD_SIZE), + ) + .expect("read index") }; // Simulate crash mid-defrag: write uncommitted WAL entry @@ -1276,7 +1300,12 @@ fn test_defragment_crash_overlapping_move_recovers() { let (pre_defrag_index, b_old_offset, b_size) = { let handle = vault_open(path.clone(), test_key()).expect("open"); let mut f = File::open(&path).expect("open file"); - let idx = read_encrypted_index(&mut f, PRIMARY_INDEX_OFFSET, encrypted_index_size(MIN_INDEX_PAD_SIZE)).expect("read idx"); + let idx = read_encrypted_index( + &mut f, + PRIMARY_INDEX_OFFSET, + encrypted_index_size(MIN_INDEX_PAD_SIZE), + ) + .expect("read idx"); let entry = handle.index.find("b.txt").expect("B"); let off = entry.offset; let sz = entry.size; @@ -1297,8 +1326,10 @@ fn test_defragment_crash_overlapping_move_recovers() { // Read B from old position let mut buf = vec![0u8; b_size as usize]; - f.seek(SeekFrom::Start(format::data_region_offset(MIN_INDEX_PAD_SIZE) + b_old_offset)) - .expect("seek"); + f.seek(SeekFrom::Start( + format::data_region_offset(MIN_INDEX_PAD_SIZE) + b_old_offset, + )) + .expect("seek"); f.read_exact(&mut buf).expect("read B"); // Write defrag backup (simulating what vault_defragment does) @@ -1310,7 +1341,10 @@ fn test_defragment_crash_overlapping_move_recovers() { backup.sync_all().expect("sync backup"); // Write B to new position (offset 0) — corrupts overlap zone - f.seek(SeekFrom::Start(format::data_region_offset(MIN_INDEX_PAD_SIZE))).expect("seek 0"); + f.seek(SeekFrom::Start(format::data_region_offset( + MIN_INDEX_PAD_SIZE, + ))) + .expect("seek 0"); f.write_all(&buf).expect("write B to 0"); f.sync_all().expect("sync"); @@ -1586,7 +1620,10 @@ fn test_vault_health_after_write_delete_and_defrag() { let after_defrag = vault_health(&handle); assert_eq!(after_defrag.free_region_count, 0); assert_eq!(after_defrag.fragmentation_ratio, 0.0); - assert_eq!(after_defrag.largest_free_block, after_defrag.unallocated_bytes); + assert_eq!( + after_defrag.largest_free_block, + after_defrag.unallocated_bytes + ); assert_eq!( after_defrag.used_bytes + after_defrag.free_list_bytes + after_defrag.unallocated_bytes, handle.index.capacity @@ -1634,5 +1671,8 @@ fn test_vault_health_full_capacity() { let h = vault_health(&handle); assert_eq!(h.free_region_count, 0); assert_eq!(h.fragmentation_ratio, 0.0); - assert_eq!(h.used_bytes + h.free_list_bytes + h.unallocated_bytes, h.total_bytes); + assert_eq!( + h.used_bytes + h.free_list_bytes + h.unallocated_bytes, + h.total_bytes + ); } From a1da50f4d925320d1e154482a634b16a93a6d4a4 Mon Sep 17 00:00:00 2001 From: Adel HB Date: Sat, 21 Mar 2026 17:09:12 +0100 Subject: [PATCH 09/21] feat(evfs): add tests for new streaming read function --- rust/src/api/evfs/tests.rs | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index adb3faf..1784980 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -1676,3 +1676,42 @@ fn test_vault_health_full_capacity() { h.total_bytes ); } + +// -- Streaming Read Tests ----------------------------------------------- +#[test] +fn test_stream_read_monolithic_interop() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let data = b"monolithic data".to_vec(); + vault_write(&mut handle, "mono.txt".into(), data.clone(), None).expect("write"); + + // Mock StreamSinks (Using FRB's testing utilities or a dummy sink if available) + // Note: Since StreamSink is hard to mock in pure Rust unit tests without FRB's test + // harness, ensure you are testing the raw Rust logic or using an FRB mock sink. + + // As a pure Rust alternative for the test suite, you can verify the chunk_count metadata: + let entry = handle.index.find("mono.txt").expect("find"); + assert_eq!(entry.chunk_count, 0, "Should be written as monolithic"); + + let read_back = vault_read(&mut handle, "mono.txt".into()).expect("read"); + assert_eq!(read_back, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_tamper_with_chunk_detected() { + let dir = tempfile::tempdir().expect("tempdir"); + let handle = create_test_vault(&dir, 1_048_576); // Removed `mut` + + // Simulate writing a chunked segment (or use stream_write if already implemented) + // For the sake of the test plan: + let _data = vec![0xAA; crate::core::streaming::CHUNK_SIZE * 2]; // Prefixed with `_` + + // NOTE: Once vault_write_stream is merged (from PR #89), use it here to write + // the chunked data, then tamper with the file directly via handle.file.seek() + // and flipping a byte, exactly like the existing `test_read_tampered_segment`. + + vault_close(handle).expect("close"); +} From 31963657a1b6cb92259ae6a6922d69c1f121bc70 Mon Sep 17 00:00:00 2001 From: Adel HB Date: Sat, 21 Mar 2026 17:13:16 +0100 Subject: [PATCH 10/21] fix(clippy): replace unwraps and remove unused/mut in tests --- lib/src/rust/api/evfs.dart | 15 +++++ lib/src/rust/frb_generated.dart | 91 ++++++++++++++++++++++++++++- lib/src/rust/frb_generated.io.dart | 16 +++++ lib/src/rust/frb_generated.web.dart | 16 +++++ rust/src/core/evfs/format.rs | 86 ++++++++++++++++++--------- rust/src/core/evfs/segment.rs | 10 +++- rust/src/frb_generated.rs | 89 +++++++++++++++++++++++++++- 7 files changed, 288 insertions(+), 35 deletions(-) diff --git a/lib/src/rust/api/evfs.dart b/lib/src/rust/api/evfs.dart index b2ad76e..bf8421f 100644 --- a/lib/src/rust/api/evfs.dart +++ b/lib/src/rust/api/evfs.dart @@ -53,6 +53,21 @@ Future vaultRead({ required String name, }) => RustLib.instance.api.crateApiEvfsVaultRead(handle: handle, name: name); +/// Read a named segment sequentially through a stream. Decompression is automatic. +Future vaultReadStream({ + required VaultHandle handle, + required String name, + required bool verifyChecksum, + required RustStreamSink sink, + required RustStreamSink onProgress, +}) => RustLib.instance.api.crateApiEvfsVaultReadStream( + handle: handle, + name: name, + verifyChecksum: verifyChecksum, + sink: sink, + onProgress: onProgress, +); + /// Delete a named segment. The region is secure-erased and returned to the /// free list for reuse by future writes. Future vaultDelete({required VaultHandle handle, required String name}) => diff --git a/lib/src/rust/frb_generated.dart b/lib/src/rust/frb_generated.dart index 0af17c0..d57f8d8 100644 --- a/lib/src/rust/frb_generated.dart +++ b/lib/src/rust/frb_generated.dart @@ -74,7 +74,7 @@ class RustLib extends BaseEntrypoint { String get codegenVersion => '2.11.1'; @override - int get rustContentHash => 187783673; + int get rustContentHash => -397954733; static const kDefaultExternalLibraryLoaderConfig = ExternalLibraryLoaderConfig( @@ -270,6 +270,14 @@ abstract class RustLibApi extends BaseApi { required String name, }); + Future crateApiEvfsVaultReadStream({ + required VaultHandle handle, + required String name, + required bool verifyChecksum, + required RustStreamSink sink, + required RustStreamSink onProgress, + }); + Future crateApiEvfsVaultResize({ required VaultHandle handle, required BigInt newCapacity, @@ -1823,6 +1831,50 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: ["handle", "name"], ); + @override + Future crateApiEvfsVaultReadStream({ + required VaultHandle handle, + required String name, + required bool verifyChecksum, + required RustStreamSink sink, + required RustStreamSink onProgress, + }) { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_Auto_RefMut_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVaultHandle( + handle, + serializer, + ); + sse_encode_String(name, serializer); + sse_encode_bool(verifyChecksum, serializer); + sse_encode_StreamSink_list_prim_u_8_strict_Sse(sink, serializer); + sse_encode_StreamSink_f_64_Sse(onProgress, serializer); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 45, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_crypto_error, + ), + constMeta: kCrateApiEvfsVaultReadStreamConstMeta, + argValues: [handle, name, verifyChecksum, sink, onProgress], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateApiEvfsVaultReadStreamConstMeta => + const TaskConstMeta( + debugName: "vault_read_stream", + argNames: ["handle", "name", "verifyChecksum", "sink", "onProgress"], + ); + @override Future crateApiEvfsVaultResize({ required VaultHandle handle, @@ -1840,7 +1892,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 45, + funcId: 46, port: port_, ); }, @@ -1884,7 +1936,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 46, + funcId: 47, port: port_, ); }, @@ -2030,6 +2082,14 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { throw UnimplementedError(); } + @protected + RustStreamSink dco_decode_StreamSink_list_prim_u_8_strict_Sse( + dynamic raw, + ) { + // Codec=Dco (DartCObject based), see doc to use other codecs + throw UnimplementedError(); + } + @protected String dco_decode_String(dynamic raw) { // Codec=Dco (DartCObject based), see doc to use other codecs @@ -2383,6 +2443,14 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { throw UnimplementedError('Unreachable ()'); } + @protected + RustStreamSink sse_decode_StreamSink_list_prim_u_8_strict_Sse( + SseDeserializer deserializer, + ) { + // Codec=Sse (Serialization based), see doc to use other codecs + throw UnimplementedError('Unreachable ()'); + } + @protected String sse_decode_String(SseDeserializer deserializer) { // Codec=Sse (Serialization based), see doc to use other codecs @@ -2808,6 +2876,23 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { ); } + @protected + void sse_encode_StreamSink_list_prim_u_8_strict_Sse( + RustStreamSink self, + SseSerializer serializer, + ) { + // Codec=Sse (Serialization based), see doc to use other codecs + sse_encode_String( + self.setupAndSerialize( + codec: SseCodec( + decodeSuccessData: sse_decode_list_prim_u_8_strict, + decodeErrorData: sse_decode_AnyhowException, + ), + ), + serializer, + ); + } + @protected void sse_encode_String(String self, SseSerializer serializer) { // Codec=Sse (Serialization based), see doc to use other codecs diff --git a/lib/src/rust/frb_generated.io.dart b/lib/src/rust/frb_generated.io.dart index 4e095b0..07a4576 100644 --- a/lib/src/rust/frb_generated.io.dart +++ b/lib/src/rust/frb_generated.io.dart @@ -106,6 +106,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { @protected RustStreamSink dco_decode_StreamSink_f_64_Sse(dynamic raw); + @protected + RustStreamSink dco_decode_StreamSink_list_prim_u_8_strict_Sse( + dynamic raw, + ); + @protected String dco_decode_String(dynamic raw); @@ -246,6 +251,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { SseDeserializer deserializer, ); + @protected + RustStreamSink sse_decode_StreamSink_list_prim_u_8_strict_Sse( + SseDeserializer deserializer, + ); + @protected String sse_decode_String(SseDeserializer deserializer); @@ -408,6 +418,12 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { SseSerializer serializer, ); + @protected + void sse_encode_StreamSink_list_prim_u_8_strict_Sse( + RustStreamSink self, + SseSerializer serializer, + ); + @protected void sse_encode_String(String self, SseSerializer serializer); diff --git a/lib/src/rust/frb_generated.web.dart b/lib/src/rust/frb_generated.web.dart index 0ed1a12..8f7688a 100644 --- a/lib/src/rust/frb_generated.web.dart +++ b/lib/src/rust/frb_generated.web.dart @@ -108,6 +108,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { @protected RustStreamSink dco_decode_StreamSink_f_64_Sse(dynamic raw); + @protected + RustStreamSink dco_decode_StreamSink_list_prim_u_8_strict_Sse( + dynamic raw, + ); + @protected String dco_decode_String(dynamic raw); @@ -248,6 +253,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { SseDeserializer deserializer, ); + @protected + RustStreamSink sse_decode_StreamSink_list_prim_u_8_strict_Sse( + SseDeserializer deserializer, + ); + @protected String sse_decode_String(SseDeserializer deserializer); @@ -410,6 +420,12 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { SseSerializer serializer, ); + @protected + void sse_encode_StreamSink_list_prim_u_8_strict_Sse( + RustStreamSink self, + SseSerializer serializer, + ); + @protected void sse_encode_String(String self, SseSerializer serializer); diff --git a/rust/src/core/evfs/format.rs b/rust/src/core/evfs/format.rs index 3ba90a5..f14af03 100644 --- a/rust/src/core/evfs/format.rs +++ b/rust/src/core/evfs/format.rs @@ -90,9 +90,7 @@ pub fn streaming_segment_size(plaintext_size: u64) -> Result { let chunk_count = plaintext_size.div_ceil(CHUNK_SIZE as u64); chunk_count .checked_mul(ENCRYPTED_CHUNK_SIZE as u64) - .ok_or_else(|| { - CryptoError::InvalidParameter("streaming segment size overflows u64".into()) - }) + .ok_or_else(|| CryptoError::InvalidParameter("streaming segment size overflows u64".into())) } /// Compute the number of chunks needed for a given plaintext size. @@ -104,9 +102,8 @@ pub fn streaming_chunk_count(plaintext_size: u64) -> Result { return Ok(0); } let count = plaintext_size.div_ceil(CHUNK_SIZE as u64); - u32::try_from(count).map_err(|_| { - CryptoError::InvalidParameter("streaming chunk count exceeds u32::MAX".into()) - }) + u32::try_from(count) + .map_err(|_| CryptoError::InvalidParameter("streaming chunk count exceeds u32::MAX".into())) } // --------------------------------------------------------------------------- @@ -814,7 +811,15 @@ mod tests { #[test] fn test_segment_entry_name_empty() { - let e = SegmentEntry::new("", 0, 100, 0, dummy_checksum(0), CompressionAlgorithm::None, 0); + let e = SegmentEntry::new( + "", + 0, + 100, + 0, + dummy_checksum(0), + CompressionAlgorithm::None, + 0, + ); assert!(e.is_err()); } @@ -860,7 +865,9 @@ mod tests { #[test] fn test_segment_index_roundtrip() { let idx = make_test_index(); - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); assert_eq!(parsed.entries.len(), 2); @@ -901,7 +908,9 @@ mod tests { ) .expect("entry"), ); - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); assert_eq!(parsed.next_generation, 42); assert_eq!(parsed.entries[0].generation, 41); @@ -931,7 +940,9 @@ mod tests { .expect("entry"), ); } - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); assert_eq!(parsed.entries[0].compression, CompressionAlgorithm::Zstd); assert_eq!(parsed.entries[1].compression, CompressionAlgorithm::Brotli); @@ -953,7 +964,9 @@ mod tests { offset: 1000, size: 300, }); - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); assert_eq!(parsed.free_regions.len(), 3); assert_eq!( @@ -997,8 +1010,8 @@ mod tests { #[test] fn test_segment_index_overflow_min_pad() { let mut idx = SegmentIndex::new(512 * 1024); // 512KB → MIN_INDEX_PAD_SIZE - // Each entry with a 255-byte name uses 2 + 255 + 8 + 8 + 8 + 32 + 1 + 4 = 318 bytes. - // Index header is 32 bytes. (65536 - 32) / 318 ≈ 205 entries max. + // Each entry with a 255-byte name uses 2 + 255 + 8 + 8 + 8 + 32 + 1 + 4 = 318 bytes. + // Index header is 32 bytes. (65536 - 32) / 318 ≈ 205 entries max. for i in 0..210 { let name = format!("{:0>255}", i); idx.entries.push( @@ -1532,7 +1545,8 @@ mod tests { let moves = idx.plan_defrag(); for m in &moves { - idx.apply_move(m.entry_index, m.new_offset).expect("apply_move"); + idx.apply_move(m.entry_index, m.new_offset) + .expect("apply_move"); } idx.complete_defrag(); @@ -1589,7 +1603,9 @@ mod tests { .expect("entry"), ); - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let parsed = SegmentIndex::from_bytes(&bytes).expect("parse"); assert_eq!(parsed.entries[0].chunk_count, 0); @@ -1614,7 +1630,9 @@ mod tests { chunk_count: 5, }); - let bytes = idx.to_bytes(compute_index_size(idx.capacity)).expect("serialize"); + let bytes = idx + .to_bytes(compute_index_size(idx.capacity)) + .expect("serialize"); let result = SegmentIndex::from_bytes(&bytes); assert!(result.is_err()); let err = result.expect_err("should fail").to_string(); @@ -1631,22 +1649,30 @@ mod tests { compression: CompressionAlgorithm::None, chunk_count: 5, }); - let bytes2 = idx2.to_bytes(compute_index_size(idx2.capacity)).expect("serialize"); + let bytes2 = idx2 + .to_bytes(compute_index_size(idx2.capacity)) + .expect("serialize"); assert!(SegmentIndex::from_bytes(&bytes2).is_ok()); } #[test] fn test_streaming_segment_size_zero() { - assert_eq!(streaming_segment_size(0).unwrap(), 0); + assert_eq!(streaming_segment_size(0).expect("expected size"), 0); } #[test] fn test_streaming_segment_size_single_chunk() { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; // 1 byte → 1 chunk - assert_eq!(streaming_segment_size(1).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); + assert_eq!( + streaming_segment_size(1).expect("expected size"), + ENCRYPTED_CHUNK_SIZE as u64 + ); // Exactly one chunk - assert_eq!(streaming_segment_size(64 * 1024).unwrap(), ENCRYPTED_CHUNK_SIZE as u64); + assert_eq!( + streaming_segment_size(64 * 1024).expect("expected size"), + ENCRYPTED_CHUNK_SIZE as u64 + ); } #[test] @@ -1654,12 +1680,12 @@ mod tests { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; // 64KB + 1 byte → 2 chunks assert_eq!( - streaming_segment_size(64 * 1024 + 1).unwrap(), + streaming_segment_size(64 * 1024 + 1).expect("expected size"), 2 * ENCRYPTED_CHUNK_SIZE as u64 ); // 5 full chunks assert_eq!( - streaming_segment_size(5 * 64 * 1024).unwrap(), + streaming_segment_size(5 * 64 * 1024).expect("expected size"), 5 * ENCRYPTED_CHUNK_SIZE as u64 ); } @@ -1671,11 +1697,17 @@ mod tests { #[test] fn test_streaming_chunk_count_values() { - assert_eq!(streaming_chunk_count(0).unwrap(), 0); - assert_eq!(streaming_chunk_count(1).unwrap(), 1); - assert_eq!(streaming_chunk_count(64 * 1024).unwrap(), 1); - assert_eq!(streaming_chunk_count(64 * 1024 + 1).unwrap(), 2); - assert_eq!(streaming_chunk_count(5 * 64 * 1024).unwrap(), 5); + assert_eq!(streaming_chunk_count(0).expect("expected count"), 0); + assert_eq!(streaming_chunk_count(1).expect("expected count"), 1); + assert_eq!(streaming_chunk_count(64 * 1024).expect("expected count"), 1); + assert_eq!( + streaming_chunk_count(64 * 1024 + 1).expect("expected count"), + 2 + ); + assert_eq!( + streaming_chunk_count(5 * 64 * 1024).expect("expected count"), + 5 + ); } #[test] diff --git a/rust/src/core/evfs/segment.rs b/rust/src/core/evfs/segment.rs index 4480f90..bc3ab99 100644 --- a/rust/src/core/evfs/segment.rs +++ b/rust/src/core/evfs/segment.rs @@ -1017,9 +1017,15 @@ mod tests { let bytes = aad.to_bytes(); assert_eq!(bytes.len(), VAULT_CHUNK_AAD_SIZE); // generation = 3 LE - assert_eq!(u64::from_le_bytes(bytes[0..8].try_into().unwrap()), 3); + assert_eq!( + u64::from_le_bytes(bytes[0..8].try_into().expect("valid slice")), + 3 + ); // chunk_index = 99 LE - assert_eq!(u64::from_le_bytes(bytes[8..16].try_into().unwrap()), 99); + assert_eq!( + u64::from_le_bytes(bytes[8..16].try_into().expect("valid slice")), + 99 + ); // is_final = true assert_eq!(bytes[16], 1); } diff --git a/rust/src/frb_generated.rs b/rust/src/frb_generated.rs index 6939fa0..7254fd8 100644 --- a/rust/src/frb_generated.rs +++ b/rust/src/frb_generated.rs @@ -40,7 +40,7 @@ flutter_rust_bridge::frb_generated_boilerplate!( default_rust_auto_opaque = RustAutoOpaqueMoi, ); pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.11.1"; -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 187783673; +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -397954733; // Section: executor @@ -1952,6 +1952,73 @@ fn wire__crate__api__evfs__vault_read_impl( }, ) } +fn wire__crate__api__evfs__vault_read_stream_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "vault_read_stream", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_handle = , + >>::sse_decode(&mut deserializer); + let api_name = ::sse_decode(&mut deserializer); + let api_verify_checksum = ::sse_decode(&mut deserializer); + let api_sink = + , flutter_rust_bridge::for_generated::SseCodec>>::sse_decode( + &mut deserializer, + ); + let api_on_progress = + >::sse_decode( + &mut deserializer, + ); + deserializer.end(); + move |context| { + transform_result_sse::<_, crate::core::error::CryptoError>((move || { + let mut api_handle_guard = None; + let decode_indices_ = + flutter_rust_bridge::for_generated::lockable_compute_decode_order(vec![ + flutter_rust_bridge::for_generated::LockableOrderInfo::new( + &api_handle, + 0, + true, + ), + ]); + for i in decode_indices_ { + match i { + 0 => api_handle_guard = Some(api_handle.lockable_decode_sync_ref_mut()), + _ => unreachable!(), + } + } + let mut api_handle_guard = api_handle_guard.unwrap(); + let output_ok = crate::api::evfs::vault_read_stream( + &mut *api_handle_guard, + api_name, + api_verify_checksum, + api_sink, + api_on_progress, + )?; + Ok(output_ok) + })()) + } + }, + ) +} fn wire__crate__api__evfs__vault_resize_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -2156,6 +2223,14 @@ impl SseDecode for StreamSink } } +impl SseDecode for StreamSink, flutter_rust_bridge::for_generated::SseCodec> { + // Codec=Sse (Serialization based), see doc to use other codecs + fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self { + let mut inner = ::sse_decode(deserializer); + return StreamSink::deserialize(inner); + } +} + impl SseDecode for String { // Codec=Sse (Serialization based), see doc to use other codecs fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self { @@ -2581,8 +2656,9 @@ fn pde_ffi_dispatcher_primary_impl( 42 => wire__crate__api__evfs__vault_list_impl(port, ptr, rust_vec_len, data_len), 43 => wire__crate__api__evfs__vault_open_impl(port, ptr, rust_vec_len, data_len), 44 => wire__crate__api__evfs__vault_read_impl(port, ptr, rust_vec_len, data_len), - 45 => wire__crate__api__evfs__vault_resize_impl(port, ptr, rust_vec_len, data_len), - 46 => wire__crate__api__evfs__vault_write_impl(port, ptr, rust_vec_len, data_len), + 45 => wire__crate__api__evfs__vault_read_stream_impl(port, ptr, rust_vec_len, data_len), + 46 => wire__crate__api__evfs__vault_resize_impl(port, ptr, rust_vec_len, data_len), + 47 => wire__crate__api__evfs__vault_write_impl(port, ptr, rust_vec_len, data_len), _ => unreachable!(), } } @@ -2916,6 +2992,13 @@ impl SseEncode for StreamSink } } +impl SseEncode for StreamSink, flutter_rust_bridge::for_generated::SseCodec> { + // Codec=Sse (Serialization based), see doc to use other codecs + fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) { + unimplemented!("") + } +} + impl SseEncode for String { // Codec=Sse (Serialization based), see doc to use other codecs fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) { From 26efb283913f0b823d5a9b88be158c8ae2eaf8f9 Mon Sep 17 00:00:00 2001 From: Adel HB Date: Sat, 21 Mar 2026 18:01:36 +0100 Subject: [PATCH 11/21] fix(evfs): correct streaming chunk count & add comprehensive stream read/write tests --- lib/src/rust/api/evfs.dart | 6 +- rust/src/api/evfs/tests.rs | 226 +++++++++++++++++++++++++++-------- rust/src/core/evfs/format.rs | 46 +++++-- 3 files changed, 213 insertions(+), 65 deletions(-) diff --git a/lib/src/rust/api/evfs.dart b/lib/src/rust/api/evfs.dart index bf8421f..0f211ef 100644 --- a/lib/src/rust/api/evfs.dart +++ b/lib/src/rust/api/evfs.dart @@ -9,7 +9,8 @@ import 'compression.dart'; import 'evfs/types.dart'; import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart'; -// These functions are ignored because they are not marked as `pub`: `vault_resize_grow_impl`, `vault_resize_shrink_impl` +// These functions are ignored because they are not marked as `pub`: `chunk_abs_offset`, `vault_resize_grow_impl`, `vault_resize_shrink_impl`, `write_encrypted_chunk` +// These functions have error during generation (see debug logs or enable `stop_on_error: true` for more details): `vault_write_stream` /// Create a new vault file at `path` with the given capacity. /// @@ -47,7 +48,8 @@ Future vaultWrite({ compression: compression, ); -/// Read a named segment. Decompression is automatic. +/// Read a named segment. Handles both monolithic and streaming segments. +/// Decompression is automatic for monolithic segments. Future vaultRead({ required VaultHandle handle, required String name, diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index 2a22b56..80ea60b 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -1677,20 +1677,15 @@ fn test_vault_health_full_capacity() { ); } -// -- Streaming Read Tests ----------------------------------------------- +// -- Streaming Read & Interop Tests ------------------------------------- #[test] -fn test_stream_read_monolithic_interop() { +fn test_oneshot_write_oneshot_read_interop() { let dir = tempfile::tempdir().expect("tempdir"); let mut handle = create_test_vault(&dir, 1_048_576); let data = b"monolithic data".to_vec(); vault_write(&mut handle, "mono.txt".into(), data.clone(), None).expect("write"); - // Mock StreamSinks (Using FRB's testing utilities or a dummy sink if available) - // Note: Since StreamSink is hard to mock in pure Rust unit tests without FRB's test - // harness, ensure you are testing the raw Rust logic or using an FRB mock sink. - - // As a pure Rust alternative for the test suite, you can verify the chunk_count metadata: let entry = handle.index.find("mono.txt").expect("find"); assert_eq!(entry.chunk_count, 0, "Should be written as monolithic"); @@ -1700,18 +1695,147 @@ fn test_stream_read_monolithic_interop() { vault_close(handle).expect("close"); } +#[test] +fn test_stream_write_oneshot_read_matches_interop() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 5_000_000); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + // 3 full chunks, 1 partial padded chunk + let data = vec![0x77; chunk_size * 3 + 1234]; + + // Write using stream + let chunks: Vec> = data.chunks(chunk_size).map(|c| c.to_vec()).collect(); + vault_write_stream( + &mut handle, + "streamed.bin".into(), + data.len() as u64, + chunks.into_iter(), + ) + .expect("stream write"); + + let entry = handle.index.find("streamed.bin").expect("find"); + assert!(entry.chunk_count > 0, "Should be written as chunked"); + + // Read using one-shot (interop) testing the chunk-assembly loop + let read_back = vault_read(&mut handle, "streamed.bin".into()).expect("read"); + assert_eq!(read_back, data, "Data should match byte-for-byte"); + + vault_close(handle).expect("close"); +} + #[test] fn test_tamper_with_chunk_detected() { let dir = tempfile::tempdir().expect("tempdir"); - let handle = create_test_vault(&dir, 1_048_576); // Removed `mut` + let mut handle = create_test_vault(&dir, 1_048_576); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + let data = vec![0xAA; chunk_size * 2]; + + let chunks = vec![data[..chunk_size].to_vec(), data[chunk_size..].to_vec()]; + vault_write_stream( + &mut handle, + "streamed.txt".into(), + data.len() as u64, + chunks.into_iter(), + ) + .expect("stream write"); + + let entry = handle.index.find("streamed.txt").expect("find"); + let disk_offset = + crate::core::evfs::format::data_region_offset(handle.index_pad_size) + entry.offset; + + // Seek past nonce (12 bytes) and flip a ciphertext byte in the first chunk + handle + .file + .seek(SeekFrom::Start(disk_offset + 13)) + .expect("seek"); + handle.file.write_all(&[0xFF]).expect("tamper"); + handle.file.sync_all().expect("sync"); + + let result = vault_read(&mut handle, "streamed.txt".into()); + assert!( + matches!(result, Err(CryptoError::AuthenticationFailed)), + "Should detect chunk tampering via independent AEAD" + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_reordered_chunks_detected() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + let enc_chunk_size = crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64; + let data = vec![0xBB; chunk_size * 2]; + + let chunks = vec![data[..chunk_size].to_vec(), data[chunk_size..].to_vec()]; + vault_write_stream( + &mut handle, + "reorder.bin".into(), + data.len() as u64, + chunks.into_iter(), + ) + .expect("stream write"); + + let entry = handle.index.find("reorder.bin").expect("find"); + let disk_offset = + crate::core::evfs::format::data_region_offset(handle.index_pad_size) + entry.offset; + + // Read chunk 0 and chunk 1 + let mut c0 = vec![0u8; enc_chunk_size as usize]; + let mut c1 = vec![0u8; enc_chunk_size as usize]; + + handle + .file + .seek(SeekFrom::Start(disk_offset)) + .expect("seek"); + handle.file.read_exact(&mut c0).expect("read c0"); + handle.file.read_exact(&mut c1).expect("read c1"); + + // Swap them on disk + handle + .file + .seek(SeekFrom::Start(disk_offset)) + .expect("seek"); + handle.file.write_all(&c1).expect("write c1"); + handle.file.write_all(&c0).expect("write c0"); + handle.file.sync_all().expect("sync"); + + let result = vault_read(&mut handle, "reorder.bin".into()); + assert!( + matches!(result, Err(CryptoError::AuthenticationFailed)), + "Should detect chunk reordering due to index mismatch in AAD" + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_integrity_checksum_failure_on_stream() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); - // Simulate writing a chunked segment (or use stream_write if already implemented) - // For the sake of the test plan: - let _data = vec![0xAA; crate::core::streaming::CHUNK_SIZE * 2]; // Prefixed with `_` + let data = vec![0xCC; 1000]; + vault_write_stream( + &mut handle, + "checksum.bin".into(), + data.len() as u64, + vec![data].into_iter(), + ) + .expect("write"); + + // Manually corrupt the checksum stored in the Segment Index + let entry = handle.index.find_mut("checksum.bin").expect("find"); + entry.checksum[0] ^= 0xFF; - // NOTE: Once vault_write_stream is merged (from PR #89), use it here to write - // the chunked data, then tamper with the file directly via handle.file.seek() - // and flipping a byte, exactly like the existing `test_read_tampered_segment`. + let result = vault_read(&mut handle, "checksum.bin".into()); + assert!( + matches!(result, Err(CryptoError::VaultCorrupted(_))), + "Should detect BLAKE3 checksum mismatch even if AEAD passes" + ); vault_close(handle).expect("close"); } @@ -1725,10 +1849,7 @@ fn stream_write_chunks( data: &[u8], piece_size: usize, ) -> Result<(), CryptoError> { - let chunks: Vec> = data - .chunks(piece_size) - .map(|c| c.to_vec()) - .collect(); + let chunks: Vec> = data.chunks(piece_size).map(|c| c.to_vec()).collect(); vault_write_stream( handle, name.to_string(), @@ -1772,13 +1893,8 @@ fn test_stream_write_empty_segment() { let mut handle = create_test_vault(&dir, 1_048_576); // 0 bytes — still produces 1 padded chunk - vault_write_stream( - &mut handle, - "empty.bin".into(), - 0, - std::iter::empty(), - ) - .expect("stream write empty"); + vault_write_stream(&mut handle, "empty.bin".into(), 0, std::iter::empty()) + .expect("stream write empty"); let readback = vault_read(&mut handle, "empty.bin".into()).expect("read"); assert!(readback.is_empty()); @@ -1861,14 +1977,11 @@ fn test_stream_write_wrong_size_too_few_bytes() { // Claim 1000 bytes but provide only 500 let data = vec![0xAA; 500]; - let result = vault_write_stream( - &mut handle, - "bad.bin".into(), - 1000, - vec![data].into_iter(), - ); + let result = vault_write_stream(&mut handle, "bad.bin".into(), 1000, vec![data].into_iter()); assert!(result.is_err()); - let err = result.unwrap_err().to_string(); + let err = result + .expect_err("expected an error (underflow)") + .to_string(); assert!(err.contains("underflow"), "Expected underflow error: {err}"); } @@ -1879,14 +1992,11 @@ fn test_stream_write_wrong_size_too_many_bytes() { // Claim 500 bytes but provide 1000 let data = vec![0xAA; 1000]; - let result = vault_write_stream( - &mut handle, - "bad.bin".into(), - 500, - vec![data].into_iter(), - ); + let result = vault_write_stream(&mut handle, "bad.bin".into(), 500, vec![data].into_iter()); assert!(result.is_err()); - let err = result.unwrap_err().to_string(); + let err = result + .expect_err("expected an error (exceeded)") + .to_string(); assert!(err.contains("exceeded"), "Expected exceeded error: {err}"); } @@ -1897,9 +2007,13 @@ fn test_stream_write_persist_reopen() { let data = vec![0xDD; 150_000]; { - let mut handle = - vault_create(path.clone(), test_key(), "aes-256-gcm".into(), 2 * 1024 * 1024) - .expect("create"); + let mut handle = vault_create( + path.clone(), + test_key(), + "aes-256-gcm".into(), + 2 * 1024 * 1024, + ) + .expect("create"); stream_write_chunks(&mut handle, "persist.bin", &data, 4096).expect("stream write"); vault_close(handle).expect("close"); } @@ -1921,9 +2035,13 @@ fn test_stream_write_chacha20() { .to_str() .expect("path") .to_string(); - let mut handle = - vault_create(path, test_key(), "chacha20-poly1305".into(), 2 * 1024 * 1024) - .expect("create"); + let mut handle = vault_create( + path, + test_key(), + "chacha20-poly1305".into(), + 2 * 1024 * 1024, + ) + .expect("create"); let data = vec![0xEE; 100_000]; stream_write_chunks(&mut handle, "chacha.bin", &data, 8192).expect("stream write"); @@ -1972,13 +2090,11 @@ fn test_stream_write_coexists_with_monolithic() { let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); // Write monolithic - vault_write(&mut handle, "mono.txt".into(), b"mono data".to_vec(), None) - .expect("write mono"); + vault_write(&mut handle, "mono.txt".into(), b"mono data".to_vec(), None).expect("write mono"); // Write streaming let stream_data = vec![0xFF; 80_000]; - stream_write_chunks(&mut handle, "stream.bin", &stream_data, 4096) - .expect("stream write"); + stream_write_chunks(&mut handle, "stream.bin", &stream_data, 4096).expect("stream write"); // Read both back let mono = vault_read(&mut handle, "mono.txt".into()).expect("read mono"); @@ -1988,8 +2104,16 @@ fn test_stream_write_coexists_with_monolithic() { assert_eq!(stream, stream_data); // Verify types - assert!(!handle.index.find("mono.txt").unwrap().is_streaming()); - assert!(handle.index.find("stream.bin").unwrap().is_streaming()); + assert!(!handle + .index + .find("mono.txt") + .expect("mono.txt missing") + .is_streaming()); + assert!(handle + .index + .find("stream.bin") + .expect("stream.bin missing") + .is_streaming()); vault_close(handle).expect("close"); } diff --git a/rust/src/core/evfs/format.rs b/rust/src/core/evfs/format.rs index 9737f82..4911151 100644 --- a/rust/src/core/evfs/format.rs +++ b/rust/src/core/evfs/format.rs @@ -80,9 +80,9 @@ pub fn total_vault_size(capacity: u64, index_pad_size: usize) -> Result Result { @@ -103,15 +103,29 @@ pub fn streaming_segment_size(plaintext_size: u64) -> Result { /// Returns `Err` if the count exceeds `u32::MAX`. pub fn streaming_chunk_count(plaintext_size: u64) -> Result { use crate::core::streaming::CHUNK_SIZE; + + // Empty plaintext => one padded chunk. if plaintext_size == 0 { return Ok(1); } - let base = plaintext_size.div_ceil(CHUNK_SIZE as u64); - let count = if plaintext_size.is_multiple_of(CHUNK_SIZE as u64) { - base + 1 + + let chunk_size = CHUNK_SIZE as u64; + + // Compute ceil division safely: (plaintext_size + chunk_size - 1) / chunk_size + let base = plaintext_size + .checked_add(chunk_size - 1) + .ok_or_else(|| CryptoError::InvalidParameter("streaming chunk count overflow".into()))? + / chunk_size; + + // If plaintext is exactly a multiple of CHUNK_SIZE, add an extra empty padded + // chunk (protocol convention). Otherwise `base` is already the correct count. + let count = if plaintext_size.is_multiple_of(chunk_size) { + base.checked_add(1) + .ok_or_else(|| CryptoError::InvalidParameter("streaming chunk count overflow".into()))? } else { base }; + u32::try_from(count) .map_err(|_| CryptoError::InvalidParameter("streaming chunk count exceeds u32::MAX".into())) } @@ -1667,7 +1681,12 @@ mod tests { #[test] fn test_streaming_segment_size_zero() { - assert_eq!(streaming_segment_size(0).expect("expected size"), 0); + use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; + // Empty segment produces one padded encrypted chunk + assert_eq!( + streaming_segment_size(0).expect("expected size"), + ENCRYPTED_CHUNK_SIZE as u64 + ); } #[test] @@ -1679,9 +1698,10 @@ mod tests { ENCRYPTED_CHUNK_SIZE as u64 ); // Exactly one chunk + // Exactly CHUNK_SIZE plaintext triggers an extra empty padded chunk => 2 chunks assert_eq!( streaming_segment_size(64 * 1024).expect("expected size"), - ENCRYPTED_CHUNK_SIZE as u64 + 2 * ENCRYPTED_CHUNK_SIZE as u64 ); } @@ -1696,7 +1716,7 @@ mod tests { // 5 * 64KB → 5 full + 1 empty padded = 6 chunks assert_eq!( streaming_segment_size(5 * 64 * 1024).expect("expected size"), - 5 * ENCRYPTED_CHUNK_SIZE as u64 + 6 * ENCRYPTED_CHUNK_SIZE as u64 ); } @@ -1707,16 +1727,18 @@ mod tests { #[test] fn test_streaming_chunk_count_values() { - assert_eq!(streaming_chunk_count(0).expect("expected count"), 0); + // empty => one padded chunk + assert_eq!(streaming_chunk_count(0).expect("expected count"), 1); assert_eq!(streaming_chunk_count(1).expect("expected count"), 1); - assert_eq!(streaming_chunk_count(64 * 1024).expect("expected count"), 1); + assert_eq!(streaming_chunk_count(64 * 1024).expect("expected count"), 2); assert_eq!( streaming_chunk_count(64 * 1024 + 1).expect("expected count"), 2 ); + // exact multiples add an extra padded empty chunk assert_eq!( streaming_chunk_count(5 * 64 * 1024).expect("expected count"), - 5 + 6 ); } From dde5a40574739acee002d3a6f9878368f685d828 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Sun, 22 Mar 2026 10:24:09 +0100 Subject: [PATCH 12/21] refactor(evfs): extract decrypt_streaming_chunks + add stream read tests --- rust/src/api/evfs/helpers.rs | 87 ++++++++++++ rust/src/api/evfs/mod.rs | 178 +++++------------------- rust/src/api/evfs/tests.rs | 262 +++++++++++++++++++++++++++++++++++ 3 files changed, 385 insertions(+), 142 deletions(-) diff --git a/rust/src/api/evfs/helpers.rs b/rust/src/api/evfs/helpers.rs index 92c6844..0d8f574 100644 --- a/rust/src/api/evfs/helpers.rs +++ b/rust/src/api/evfs/helpers.rs @@ -5,6 +5,11 @@ use crate::core::format::Algorithm; use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; +#[cfg(feature = "compression")] +use crate::api::compression::CompressionAlgorithm; +#[cfg(feature = "compression")] +use subtle::ConstantTimeEq; + pub(crate) fn parse_algorithm(s: &str) -> Result { match s { "aes-256-gcm" => Ok(Algorithm::AesGcm), @@ -89,3 +94,85 @@ pub(crate) fn capacity_from_file_size( .checked_sub(overhead) .ok_or_else(|| CryptoError::VaultCorrupted("vault file too small".into())) } + +/// Decrypt all chunks of a streaming segment, calling `on_chunk(plaintext, chunk_index)` +/// for each decrypted chunk. Returns the BLAKE3 checksum of the full decrypted data. +#[allow(clippy::too_many_arguments)] +#[cfg(feature = "compression")] +pub(crate) fn decrypt_streaming_chunks( + file: &mut File, + cipher_key: &[u8], + nonce_key: &[u8], + algorithm: Algorithm, + index_pad_size: usize, + seg_offset: u64, + generation: u64, + compression: CompressionAlgorithm, + chunk_count: u32, + mut on_chunk: impl FnMut(Vec, u32) -> Result<(), CryptoError>, +) -> Result<[u8; 32], CryptoError> { + let data_region = format::data_region_offset(index_pad_size); + let mut hasher = blake3::Hasher::new(); + let mut decompressor = if compression != CompressionAlgorithm::None { + Some(crate::core::compression::streaming::new_decompressor( + compression, + )?) + } else { + None + }; + let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); + + for i in 0..chunk_count { + let chunk_offset = data_region + + seg_offset + + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); + file.seek(SeekFrom::Start(chunk_offset))?; + + let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; + file.read_exact(&mut encrypted)?; + + let expected_nonce = segment::derive_chunk_nonce(nonce_key, i as u64, generation)?; + let (stored_nonce, _) = encrypted.split_at(crate::core::streaming::NONCE_SIZE); + if stored_nonce.ct_ne(&expected_nonce).into() { + return Err(CryptoError::AuthenticationFailed); + } + + let is_final = i == chunk_count - 1; + let aad = segment::VaultChunkAad { + generation, + chunk_index: i as u64, + is_final, + } + .to_bytes(); + + let decrypted = segment::aead_decrypt_with_stored_nonce( + cipher_key, + &encrypted, + &aad, + algorithm, + )?; + + let plaintext = if is_final { + crate::core::streaming::strip_last_chunk_padding(&decrypted)? + } else { + decrypted + }; + + let final_data = if let Some(ref mut dec) = decompressor { + dec.decompress_chunk(&plaintext, &mut decomp_buf)?; + if is_final { + dec.finish(&mut decomp_buf)?; + } + let data = decomp_buf.clone(); + decomp_buf.clear(); + data + } else { + plaintext + }; + + hasher.update(&final_data); + on_chunk(final_data, i)?; + } + + Ok(hasher.finalize().into()) +} diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index d4e0898..eb15ecc 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -347,77 +347,24 @@ pub fn vault_read(handle: &mut VaultHandle, name: String) -> Result, Cry // INTEROP: If the segment is chunked, reassemble it into a single vector if chunk_count > 0 { - let data_region = format::data_region_offset(handle.index_pad_size); - let mut hasher = blake3::Hasher::new(); - let mut decompressor = if seg_compression != CompressionAlgorithm::None { - Some(crate::core::compression::streaming::new_decompressor( - seg_compression, - )?) - } else { - None - }; - - let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); let mut full_plaintext = Vec::new(); + let checksum = decrypt_streaming_chunks( + &mut handle.file, + handle.keys.cipher_key.as_bytes(), + handle.keys.nonce_key.as_bytes(), + handle.algorithm, + handle.index_pad_size, + seg_offset, + seg_gen, + seg_compression, + chunk_count, + |data, _| { + full_plaintext.extend_from_slice(&data); + Ok(()) + }, + )?; - for i in 0..chunk_count { - let chunk_offset = data_region - + seg_offset - + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); - handle.file.seek(SeekFrom::Start(chunk_offset))?; - - let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; - handle.file.read_exact(&mut encrypted)?; - - // Verify independent chunk nonce - let expected_nonce = - segment::derive_chunk_nonce(handle.keys.nonce_key.as_bytes(), i as u64, seg_gen)?; - let (stored_nonce, _) = encrypted.split_at(crate::core::streaming::NONCE_SIZE); - if stored_nonce.ct_ne(&expected_nonce).into() { - return Err(CryptoError::AuthenticationFailed); - } - - let is_final = i == chunk_count - 1; - let aad = segment::VaultChunkAad { - generation: seg_gen, - chunk_index: i as u64, - is_final, - } - .to_bytes(); - - let decrypted = segment::aead_decrypt_with_stored_nonce( - handle.keys.cipher_key.as_bytes(), - &encrypted, - &aad, - handle.algorithm, - )?; - - // Strip padding purely from the final chunk - let plaintext = if is_final { - crate::core::streaming::strip_last_chunk_padding(&decrypted)? - } else { - decrypted - }; - - let final_data = if let Some(ref mut dec) = decompressor { - dec.decompress_chunk(&plaintext, &mut decomp_buf)?; - if is_final { - dec.finish(&mut decomp_buf)?; - } - let data = decomp_buf.clone(); - decomp_buf.clear(); - data - } else { - plaintext - }; - - hasher.update(&final_data); - full_plaintext.extend_from_slice(&final_data); - } - - // Verify full file integrity using BLAKE3 Checksum - let actual_checksum = hasher.finalize(); - if actual_checksum.as_bytes().ct_ne(&seg_checksum).into() { + if checksum.ct_ne(&seg_checksum).into() { return Err(CryptoError::VaultCorrupted(format!( "integrity check failed for segment '{name}'" ))); @@ -486,80 +433,27 @@ pub fn vault_read_stream( return Ok(()); } - let data_region = format::data_region_offset(handle.index_pad_size); - let mut hasher = blake3::Hasher::new(); - let mut decompressor = if seg_compression != CompressionAlgorithm::None { - Some(crate::core::compression::streaming::new_decompressor( - seg_compression, - )?) - } else { - None - }; - - let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); - - for i in 0..chunk_count { - let chunk_offset = data_region - + seg_offset - + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); - handle.file.seek(SeekFrom::Start(chunk_offset))?; - - let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; - handle.file.read_exact(&mut encrypted)?; - - let expected_nonce = - segment::derive_chunk_nonce(handle.keys.nonce_key.as_bytes(), i as u64, seg_gen)?; - - let (stored_nonce, _) = encrypted.split_at(crate::core::streaming::NONCE_SIZE); - if stored_nonce.ct_ne(&expected_nonce).into() { - return Err(CryptoError::AuthenticationFailed); - } - - let is_final = i == chunk_count - 1; - let aad = segment::VaultChunkAad { - generation: seg_gen, - chunk_index: i as u64, - is_final, - } - .to_bytes(); - - let decrypted = segment::aead_decrypt_with_stored_nonce( - handle.keys.cipher_key.as_bytes(), - &encrypted, - &aad, - handle.algorithm, - )?; - - let plaintext = if is_final { - crate::core::streaming::strip_last_chunk_padding(&decrypted)? - } else { - decrypted - }; - - let final_data = if let Some(ref mut dec) = decompressor { - dec.decompress_chunk(&plaintext, &mut decomp_buf)?; - if is_final { - dec.finish(&mut decomp_buf)?; - } - let data = decomp_buf.clone(); - decomp_buf.clear(); - data - } else { - plaintext - }; - - hasher.update(&final_data); - let _ = sink.add(final_data); - let _ = on_progress.add(((i + 1) as f64) / (chunk_count as f64)); - } + let checksum = decrypt_streaming_chunks( + &mut handle.file, + handle.keys.cipher_key.as_bytes(), + handle.keys.nonce_key.as_bytes(), + handle.algorithm, + handle.index_pad_size, + seg_offset, + seg_gen, + seg_compression, + chunk_count, + |data, i| { + let _ = sink.add(data); + let _ = on_progress.add(((i + 1) as f64) / (chunk_count as f64)); + Ok(()) + }, + )?; - if verify_checksum { - let actual_checksum = hasher.finalize(); - if actual_checksum.as_bytes().ct_ne(&seg_checksum).into() { - return Err(CryptoError::VaultCorrupted(format!( - "integrity check failed for segment '{name}'" - ))); - } + if verify_checksum && checksum.ct_ne(&seg_checksum).into() { + return Err(CryptoError::VaultCorrupted(format!( + "integrity check failed for segment '{name}'" + ))); } Ok(()) diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index 80ea60b..e1836e3 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -2117,3 +2117,265 @@ fn test_stream_write_coexists_with_monolithic() { vault_close(handle).expect("close"); } + +// -- Streaming Read (via decrypt_streaming_chunks) -------------------------- + +/// Helper: stream-read all chunks into a Vec, returning (data, chunk_indices, checksum). +fn stream_read_chunks( + handle: &mut VaultHandle, + name: &str, +) -> Result<(Vec, Vec, [u8; 32]), CryptoError> { + let entry = handle + .index + .find(name) + .ok_or_else(|| CryptoError::SegmentNotFound(name.into()))?; + + let seg_offset = entry.offset; + let seg_gen = entry.generation; + let seg_compression = entry.compression; + let chunk_count = entry.chunk_count; + + let mut collected = Vec::new(); + let mut indices = Vec::new(); + + let checksum = decrypt_streaming_chunks( + &mut handle.file, + handle.keys.cipher_key.as_bytes(), + handle.keys.nonce_key.as_bytes(), + handle.algorithm, + handle.index_pad_size, + seg_offset, + seg_gen, + seg_compression, + chunk_count, + |data, i| { + collected.extend_from_slice(&data); + indices.push(i); + Ok(()) + }, + )?; + + Ok((collected, indices, checksum)) +} + +#[test] +fn test_stream_read_matches_oneshot_read() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data = vec![0x42u8; 200_000]; + stream_write_chunks(&mut handle, "video.bin", &data, 4096).expect("stream write"); + + let oneshot = vault_read(&mut handle, "video.bin".into()).expect("oneshot read"); + let (streamed, _, _) = stream_read_chunks(&mut handle, "video.bin").expect("stream read"); + + assert_eq!(streamed, oneshot, "streaming and one-shot must be byte-identical"); + assert_eq!(streamed, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_checksum_matches_stored() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data = vec![0xAA; 150_000]; + stream_write_chunks(&mut handle, "file.bin", &data, 8192).expect("stream write"); + + let stored_checksum = handle.index.find("file.bin").expect("find").checksum; + let (_, _, computed_checksum) = + stream_read_chunks(&mut handle, "file.bin").expect("stream read"); + + assert_eq!(computed_checksum, stored_checksum); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_progress_indices() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + let data = vec![0xBB; chunk_size * 3 + 1234]; + stream_write_chunks(&mut handle, "prog.bin", &data, chunk_size).expect("stream write"); + + let chunk_count = handle.index.find("prog.bin").expect("find").chunk_count; + let (collected, indices, _) = + stream_read_chunks(&mut handle, "prog.bin").expect("stream read"); + + assert_eq!(collected, data); + assert_eq!(indices.len(), chunk_count as usize); + let expected: Vec = (0..chunk_count).collect(); + assert_eq!(indices, expected); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_single_byte() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let data = vec![0xCC; 1]; + stream_write_chunks(&mut handle, "tiny.bin", &data, 1).expect("stream write"); + + let (collected, indices, _) = + stream_read_chunks(&mut handle, "tiny.bin").expect("stream read"); + + assert_eq!(collected, data); + assert_eq!(indices.len(), 1); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_empty_segment() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + vault_write_stream(&mut handle, "empty.bin".into(), 0, std::iter::empty()) + .expect("stream write empty"); + + let (collected, indices, _) = + stream_read_chunks(&mut handle, "empty.bin").expect("stream read"); + + assert!(collected.is_empty()); + assert_eq!(indices.len(), 1); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_exact_chunk_boundary() { + use crate::core::streaming::CHUNK_SIZE; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data = vec![0xDD; CHUNK_SIZE]; + stream_write_chunks(&mut handle, "aligned.bin", &data, CHUNK_SIZE).expect("stream write"); + + let (collected, _, checksum) = + stream_read_chunks(&mut handle, "aligned.bin").expect("stream read"); + + assert_eq!(collected, data); + let stored = handle.index.find("aligned.bin").expect("find").checksum; + assert_eq!(checksum, stored); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_large_segment() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 12 * 1024 * 1024); + + let data: Vec = (0..10_000_000).map(|i| (i % 251) as u8).collect(); + stream_write_chunks(&mut handle, "big.bin", &data, 65536).expect("stream write"); + + let (collected, indices, checksum) = + stream_read_chunks(&mut handle, "big.bin").expect("stream read"); + + assert_eq!(collected.len(), data.len()); + assert_eq!(collected, data); + assert!(!indices.is_empty()); + assert_eq!( + checksum, + handle.index.find("big.bin").expect("find").checksum + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_tamper_detected() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + let data = vec![0xEE; chunk_size * 2]; + stream_write_chunks(&mut handle, "tamper.bin", &data, chunk_size).expect("stream write"); + + let entry = handle.index.find("tamper.bin").expect("find"); + let disk_offset = + crate::core::evfs::format::data_region_offset(handle.index_pad_size) + entry.offset; + + handle + .file + .seek(std::io::SeekFrom::Start(disk_offset + 13)) + .expect("seek"); + handle.file.write_all(&[0xFF]).expect("tamper"); + handle.file.sync_all().expect("sync"); + + let result = stream_read_chunks(&mut handle, "tamper.bin"); + assert!( + matches!(result, Err(CryptoError::AuthenticationFailed)), + "Should detect chunk tampering via streaming read" + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_reorder_detected() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let chunk_size = crate::core::streaming::CHUNK_SIZE; + let enc_chunk_size = crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64; + let data = vec![0xFF; chunk_size * 2]; + stream_write_chunks(&mut handle, "reorder.bin", &data, chunk_size).expect("stream write"); + + let entry = handle.index.find("reorder.bin").expect("find"); + let disk_offset = + crate::core::evfs::format::data_region_offset(handle.index_pad_size) + entry.offset; + + let mut c0 = vec![0u8; enc_chunk_size as usize]; + let mut c1 = vec![0u8; enc_chunk_size as usize]; + handle + .file + .seek(std::io::SeekFrom::Start(disk_offset)) + .expect("seek"); + handle.file.read_exact(&mut c0).expect("read c0"); + handle.file.read_exact(&mut c1).expect("read c1"); + handle + .file + .seek(std::io::SeekFrom::Start(disk_offset)) + .expect("seek"); + handle.file.write_all(&c1).expect("write c1"); + handle.file.write_all(&c0).expect("write c0"); + handle.file.sync_all().expect("sync"); + + let result = stream_read_chunks(&mut handle, "reorder.bin"); + assert!( + matches!(result, Err(CryptoError::AuthenticationFailed)), + "Should detect chunk reordering via streaming read" + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_stream_read_chacha20() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir + .path() + .join("test.vault") + .to_str() + .expect("path") + .to_string(); + let mut handle = + vault_create(path, test_key(), "chacha20-poly1305".into(), 2 * 1024 * 1024) + .expect("create"); + + let data = vec![0x77; 200_000]; + stream_write_chunks(&mut handle, "chacha.bin", &data, 4096).expect("stream write"); + + let (collected, _, _) = + stream_read_chunks(&mut handle, "chacha.bin").expect("stream read"); + assert_eq!(collected, data); + + vault_close(handle).expect("close"); +} From e15ed42ec5762b63d757fd6e0242acf189656027 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Mon, 23 Mar 2026 14:22:24 +0100 Subject: [PATCH 13/21] feat(evfs): add vault_write_file FRB-callable wrapper for streaming write --- rust/src/api/evfs/mod.rs | 41 +++++++++++++ rust/src/api/evfs/tests.rs | 119 +++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index eb15ecc..b951210 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -603,6 +603,47 @@ pub fn vault_write_stream( Ok(()) } +/// Write a file into the vault as a streaming segment. +/// +/// Reads `file_path` in 64KB chunks and encrypts each independently. +/// This is the FRB-callable wrapper around `vault_write_stream`. +#[cfg(feature = "compression")] +pub fn vault_write_file( + handle: &mut VaultHandle, + name: String, + file_path: String, + on_progress: StreamSink, +) -> Result<(), CryptoError> { + use crate::core::streaming::CHUNK_SIZE; + + let mut file = File::open(&file_path) + .map_err(|e| CryptoError::IoError(format!("cannot open '{file_path}': {e}")))?; + + let file_size = file.seek(SeekFrom::End(0))?; + file.seek(SeekFrom::Start(0))?; + + let mut bytes_read: u64 = 0; + let data_stream = std::iter::from_fn(|| { + let mut buf = vec![0u8; CHUNK_SIZE]; + match file.read(&mut buf) { + Ok(0) => None, + Ok(n) => { + buf.truncate(n); + bytes_read += n as u64; + if file_size > 0 { + let _ = on_progress.add(bytes_read as f64 / file_size as f64); + } + Some(buf) + } + Err(_) => None, + } + }); + + vault_write_stream(handle, name, file_size, data_stream)?; + let _ = on_progress.add(1.0); + Ok(()) +} + fn chunk_abs_offset(data_off: u64, seg_offset: u64, chunk_index: u64) -> Result { use crate::core::streaming::ENCRYPTED_CHUNK_SIZE; chunk_index diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index e1836e3..7065a82 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -2379,3 +2379,122 @@ fn test_stream_read_chacha20() { vault_close(handle).expect("close"); } + +// -- vault_write_file (FRB wrapper) ----------------------------------------- + +/// Helper: write `data` to a temp file and return the path. +fn write_temp_file(dir: &tempfile::TempDir, name: &str, data: &[u8]) -> String { + let path = dir.path().join(name); + std::fs::write(&path, data).expect("write temp file"); + path.to_str().expect("path").to_string() +} + +/// Fake progress sink that collects values (vault_write_file can't use StreamSink in tests, +/// but the underlying vault_write_stream is what's actually tested here). +#[test] +fn test_write_file_roundtrip() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data = vec![0x42u8; 200_000]; + let file_path = write_temp_file(&dir, "input.bin", &data); + + // vault_write_file needs StreamSink — test the underlying path instead: + // read file → feed to vault_write_stream → read back + use crate::core::streaming::CHUNK_SIZE; + let file_data = std::fs::read(&file_path).expect("read file"); + let chunks: Vec> = file_data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); + vault_write_stream( + &mut handle, + "from_file.bin".into(), + file_data.len() as u64, + chunks.into_iter(), + ) + .expect("write stream"); + + let readback = vault_read(&mut handle, "from_file.bin".into()).expect("read"); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_write_file_stream_read_interop() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 2 * 1024 * 1024); + + let data: Vec = (0..150_000).map(|i| (i % 199) as u8).collect(); + let file_path = write_temp_file(&dir, "interop.bin", &data); + + use crate::core::streaming::CHUNK_SIZE; + let file_data = std::fs::read(&file_path).expect("read file"); + let chunks: Vec> = file_data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); + vault_write_stream( + &mut handle, + "interop.bin".into(), + file_data.len() as u64, + chunks.into_iter(), + ) + .expect("write stream"); + + // Read back via streaming read helper + let (collected, _, checksum) = + stream_read_chunks(&mut handle, "interop.bin").expect("stream read"); + assert_eq!(collected, data); + assert_eq!( + checksum, + handle.index.find("interop.bin").expect("find").checksum + ); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_write_file_large() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 12 * 1024 * 1024); + + let data: Vec = (0..5_000_000).map(|i| (i % 251) as u8).collect(); + let file_path = write_temp_file(&dir, "large.bin", &data); + + use crate::core::streaming::CHUNK_SIZE; + let file_data = std::fs::read(&file_path).expect("read file"); + let chunks: Vec> = file_data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); + vault_write_stream( + &mut handle, + "large.bin".into(), + file_data.len() as u64, + chunks.into_iter(), + ) + .expect("write stream"); + + let readback = vault_read(&mut handle, "large.bin".into()).expect("read"); + assert_eq!(readback.len(), data.len()); + assert_eq!(readback, data); + + vault_close(handle).expect("close"); +} + +#[test] +fn test_write_file_empty() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut handle = create_test_vault(&dir, 1_048_576); + + let file_path = write_temp_file(&dir, "empty.bin", &[]); + + use crate::core::streaming::CHUNK_SIZE; + let file_data = std::fs::read(&file_path).expect("read file"); + let chunks: Vec> = file_data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); + vault_write_stream( + &mut handle, + "empty.bin".into(), + 0, + chunks.into_iter(), + ) + .expect("write stream"); + + let readback = vault_read(&mut handle, "empty.bin".into()).expect("read"); + assert!(readback.is_empty()); + + vault_close(handle).expect("close"); +} From e2eb39a38a777c4b0a2c5b619f23ee137578ae38 Mon Sep 17 00:00:00 2001 From: nferhat Date: Mon, 23 Mar 2026 20:49:44 +0100 Subject: [PATCH 14/21] chore: Fix failing clippy lint in tests --- rust/src/api/evfs/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/src/api/evfs/tests.rs b/rust/src/api/evfs/tests.rs index 7065a82..c72ed83 100644 --- a/rust/src/api/evfs/tests.rs +++ b/rust/src/api/evfs/tests.rs @@ -2121,6 +2121,7 @@ fn test_stream_write_coexists_with_monolithic() { // -- Streaming Read (via decrypt_streaming_chunks) -------------------------- /// Helper: stream-read all chunks into a Vec, returning (data, chunk_indices, checksum). +#[allow(clippy::type_complexity)] fn stream_read_chunks( handle: &mut VaultHandle, name: &str, From e963b769e28f4f90adc74062d7adbed9137619a8 Mon Sep 17 00:00:00 2001 From: nferhat Date: Thu, 26 Mar 2026 12:26:47 +0100 Subject: [PATCH 15/21] dart(vault_service): Add `writeStream` implementation We first write into an intermediary file. We hand over the file name/handle to rust and it will be responsible for writing the data. --- lib/src/evfs/vault_service.dart | 90 ++++++++++++++++++++++++++++++++- lib/src/rust/api/evfs.dart | 14 +++++ lib/src/rust/frb_generated.dart | 52 ++++++++++++++++++- rust/src/frb_generated.rs | 65 +++++++++++++++++++++++- 4 files changed, 218 insertions(+), 3 deletions(-) diff --git a/lib/src/evfs/vault_service.dart b/lib/src/evfs/vault_service.dart index 1403ac8..8c5da39 100644 --- a/lib/src/evfs/vault_service.dart +++ b/lib/src/evfs/vault_service.dart @@ -1,7 +1,12 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_rust_bridge/flutter_rust_bridge.dart' + show RustStreamSink; import 'package:m_security/src/rust/api/evfs.dart' as rust_evfs; import 'package:m_security/src/rust/api/evfs/types.dart' as rust_types; import 'package:m_security/src/rust/api/compression.dart'; -import 'dart:typed_data'; /// Encrypted Virtual File System — named segment storage in a .vault container. /// @@ -54,6 +59,62 @@ class VaultService { ); } + /// Write a named segment from a Dart [Stream]. + /// + /// Pipes [data] through a temporary file on disk so that Dart RAM usage + /// is bounded to a single chunk. [totalSize] must equal the exact number + /// of bytes that [data] will emit. + static Future writeStream({ + required rust_types.VaultHandle handle, + required String name, + required int totalSize, + required Stream data, + }) async { + final tempDir = await Directory.systemTemp.createTemp('vault_write_stream'); + final tempFile = File('${tempDir.path}/payload.bin'); + + try { + // Write each incoming chunk straight to disk — only one chunk lives in + // Dart memory at a time. + final raf = await tempFile.open(mode: FileMode.writeOnly); + int bytesReceived = 0; + + try { + await for (final chunk in data) { + bytesReceived += chunk.length; + if (bytesReceived > totalSize) { + throw ArgumentError( + 'writeStream: stream overflow — ' + 'received >$bytesReceived bytes but totalSize is $totalSize', + ); + } + await raf.writeFrom(chunk); + } + } finally { + await raf.close(); + } + + if (bytesReceived != totalSize) { + throw ArgumentError( + 'writeStream: stream underflow — ' + 'received $bytesReceived bytes but totalSize is $totalSize', + ); + } + + // Delegate to vaultWriteFile which reads the temp file in 64 KB chunks + // inside Rust — keeping the end-to-end memory footprint bounded. + await _guardedStream( + () => rust_evfs.vaultWriteFile( + handle: handle, + name: name, + filePath: tempFile.path, + ), + ).drain(); + } finally { + await tempDir.delete(recursive: true); + } + } + /// Read a named segment. Decompression is automatic. static Future read({ required rust_types.VaultHandle handle, @@ -118,4 +179,31 @@ class VaultService { static Future close({required rust_types.VaultHandle handle}) { return rust_evfs.vaultClose(handle: handle); } + + // Same one from compression_service.dart + static Stream _guardedStream(Stream Function() factory) { + final controller = StreamController(); + runZonedGuarded( + () { + factory().listen( + controller.add, + onError: controller.addError, + onDone: () { + // FRB delivers zone errors after the stream closes; delay the + // controller close by one event-loop turn so they arrive first. + Future(() { + if (!controller.isClosed) controller.close(); + }); + }, + ); + }, + (Object error, StackTrace stack) { + if (!controller.isClosed) { + controller.addError(error, stack); + controller.close(); + } + }, + ); + return controller.stream; + } } diff --git a/lib/src/rust/api/evfs.dart b/lib/src/rust/api/evfs.dart index 0f211ef..224065f 100644 --- a/lib/src/rust/api/evfs.dart +++ b/lib/src/rust/api/evfs.dart @@ -70,6 +70,20 @@ Future vaultReadStream({ onProgress: onProgress, ); +/// Write a file into the vault as a streaming segment. +/// +/// Reads `file_path` in 64KB chunks and encrypts each independently. +/// This is the FRB-callable wrapper around `vault_write_stream`. +Stream vaultWriteFile({ + required VaultHandle handle, + required String name, + required String filePath, +}) => RustLib.instance.api.crateApiEvfsVaultWriteFile( + handle: handle, + name: name, + filePath: filePath, +); + /// Delete a named segment. The region is secure-erased and returned to the /// free list for reuse by future writes. Future vaultDelete({required VaultHandle handle, required String name}) => diff --git a/lib/src/rust/frb_generated.dart b/lib/src/rust/frb_generated.dart index d57f8d8..933f4a8 100644 --- a/lib/src/rust/frb_generated.dart +++ b/lib/src/rust/frb_generated.dart @@ -74,7 +74,7 @@ class RustLib extends BaseEntrypoint { String get codegenVersion => '2.11.1'; @override - int get rustContentHash => -397954733; + int get rustContentHash => 2084471439; static const kDefaultExternalLibraryLoaderConfig = ExternalLibraryLoaderConfig( @@ -290,6 +290,12 @@ abstract class RustLibApi extends BaseApi { CompressionConfig? compression, }); + Stream crateApiEvfsVaultWriteFile({ + required VaultHandle handle, + required String name, + required String filePath, + }); + RustArcIncrementStrongCountFnType get rust_arc_increment_strong_count_CipherHandle; @@ -1956,6 +1962,50 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: ["handle", "name", "data", "compression"], ); + @override + Stream crateApiEvfsVaultWriteFile({ + required VaultHandle handle, + required String name, + required String filePath, + }) { + final onProgress = RustStreamSink(); + unawaited( + handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_Auto_RefMut_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVaultHandle( + handle, + serializer, + ); + sse_encode_String(name, serializer); + sse_encode_String(filePath, serializer); + sse_encode_StreamSink_f_64_Sse(onProgress, serializer); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 48, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_crypto_error, + ), + constMeta: kCrateApiEvfsVaultWriteFileConstMeta, + argValues: [handle, name, filePath, onProgress], + apiImpl: this, + ), + ), + ); + return onProgress.stream; + } + + TaskConstMeta get kCrateApiEvfsVaultWriteFileConstMeta => const TaskConstMeta( + debugName: "vault_write_file", + argNames: ["handle", "name", "filePath", "onProgress"], + ); + RustArcIncrementStrongCountFnType get rust_arc_increment_strong_count_CipherHandle => wire .rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerCipherHandle; diff --git a/rust/src/frb_generated.rs b/rust/src/frb_generated.rs index 7254fd8..940bb94 100644 --- a/rust/src/frb_generated.rs +++ b/rust/src/frb_generated.rs @@ -40,7 +40,7 @@ flutter_rust_bridge::frb_generated_boilerplate!( default_rust_auto_opaque = RustAutoOpaqueMoi, ); pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.11.1"; -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -397954733; +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 2084471439; // Section: executor @@ -2132,6 +2132,68 @@ fn wire__crate__api__evfs__vault_write_impl( }, ) } +fn wire__crate__api__evfs__vault_write_file_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "vault_write_file", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_handle = , + >>::sse_decode(&mut deserializer); + let api_name = ::sse_decode(&mut deserializer); + let api_file_path = ::sse_decode(&mut deserializer); + let api_on_progress = + >::sse_decode( + &mut deserializer, + ); + deserializer.end(); + move |context| { + transform_result_sse::<_, crate::core::error::CryptoError>((move || { + let mut api_handle_guard = None; + let decode_indices_ = + flutter_rust_bridge::for_generated::lockable_compute_decode_order(vec![ + flutter_rust_bridge::for_generated::LockableOrderInfo::new( + &api_handle, + 0, + true, + ), + ]); + for i in decode_indices_ { + match i { + 0 => api_handle_guard = Some(api_handle.lockable_decode_sync_ref_mut()), + _ => unreachable!(), + } + } + let mut api_handle_guard = api_handle_guard.unwrap(); + let output_ok = crate::api::evfs::vault_write_file( + &mut *api_handle_guard, + api_name, + api_file_path, + api_on_progress, + )?; + Ok(output_ok) + })()) + } + }, + ) +} // Section: related_funcs @@ -2659,6 +2721,7 @@ fn pde_ffi_dispatcher_primary_impl( 45 => wire__crate__api__evfs__vault_read_stream_impl(port, ptr, rust_vec_len, data_len), 46 => wire__crate__api__evfs__vault_resize_impl(port, ptr, rust_vec_len, data_len), 47 => wire__crate__api__evfs__vault_write_impl(port, ptr, rust_vec_len, data_len), + 48 => wire__crate__api__evfs__vault_write_file_impl(port, ptr, rust_vec_len, data_len), _ => unreachable!(), } } From 9f81ab3b6e2dac7a0666a1a12be047ed758b7ab5 Mon Sep 17 00:00:00 2001 From: nferhat Date: Thu, 26 Mar 2026 12:28:31 +0100 Subject: [PATCH 16/21] dart(vault_service): Add `readStream` implementation Nothing special too. We make sure it's running in a guarded task by the Dart engine. --- lib/src/evfs/vault_service.dart | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/lib/src/evfs/vault_service.dart b/lib/src/evfs/vault_service.dart index 8c5da39..2e449a9 100644 --- a/lib/src/evfs/vault_service.dart +++ b/lib/src/evfs/vault_service.dart @@ -123,6 +123,73 @@ class VaultService { return rust_evfs.vaultRead(handle: handle, name: name); } + /// Read a named segment. Decompression bytes are sent chunk-by-chunk as they are processed by + /// the vault. + static Stream readStream({ + required rust_types.VaultHandle handle, + required String name, + }) { + final controller = StreamController(); + final dataSink = RustStreamSink(); + final progressSink = RustStreamSink(); + + runZonedGuarded( + () { + // Forward each plaintext chunk from Rust to the public stream. + dataSink.stream.listen( + controller.add, + onError: (Object e, StackTrace s) { + if (!controller.isClosed) { + controller.addError(e, s); + controller.close(); + } + }, + onDone: () { + // Schedule the close one event-loop turn later so that any + // pending error arriving from the vaultReadStream Future + // (see catchError below) can still be forwarded before the + // controller is closed. + Future(() { + if (!controller.isClosed) controller.close(); + }); + }, + cancelOnError: true, + ); + + // Drain the progress sink silently — we expose data chunks, not + // progress, through the public Stream API. + progressSink.stream.listen(null); + + // Kick off the Rust streaming read. Errors from Rust arrive via + // the returned Future (executeNormal, not unawaited), so we + // forward them with catchError. + rust_evfs + .vaultReadStream( + handle: handle, + name: name, + verifyChecksum: true, + sink: dataSink, + onProgress: progressSink, + ) + .catchError((Object e, StackTrace s) { + if (!controller.isClosed) { + controller.addError(e, s); + controller.close(); + } + }); + }, + // Safety net for any unexpected zone errors (e.g. internal FRB bugs). + (Object e, StackTrace s) { + if (!controller.isClosed) { + controller.addError(e, s); + controller.close(); + } + }, + ); + + return controller.stream; + } + /// Delete a named segment (securely erased from disk). static Future delete({ required rust_types.VaultHandle handle, From 562f93801434b6acbaeecc9288dd5407906b00af Mon Sep 17 00:00:00 2001 From: nferhat Date: Thu, 26 Mar 2026 12:30:09 +0100 Subject: [PATCH 17/21] dart(vault_service): Add integration/unit testing As described in the issue, writing given file sizes and expecting the output. --- example/integration_test/evfs_test.dart | 423 ++++++++++++++++++++---- integration_test/evfs_test.dart | 152 ++++++--- 2 files changed, 472 insertions(+), 103 deletions(-) diff --git a/example/integration_test/evfs_test.dart b/example/integration_test/evfs_test.dart index 382f72e..fc6e3ad 100644 --- a/example/integration_test/evfs_test.dart +++ b/example/integration_test/evfs_test.dart @@ -415,12 +415,12 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 5 * 1024 * 1024, // 5MB ); - + // Prepare test data (compressible - lots of repeats) final dataA = Uint8List.fromList(List.filled(5000, 42)); final dataB = Uint8List.fromList(List.filled(5000, 99)); final dataC = Uint8List.fromList(List.filled(5000, 123)); - + // Write segment A with Zstd await VaultService.write( handle: handle, @@ -431,7 +431,7 @@ void main() { level: 3, ), ); - + // Write segment B with Brotli await VaultService.write( handle: handle, @@ -442,7 +442,7 @@ void main() { level: 4, ), ); - + // Write segment C with None (no compression) await VaultService.write( handle: handle, @@ -452,22 +452,50 @@ void main() { algorithm: CompressionAlgorithm.none, ), ); - + // Read all three back - final resultA = await VaultService.read(handle: handle, name: 'zstd_segment.txt'); - final resultB = await VaultService.read(handle: handle, name: 'brotli_segment.txt'); - final resultC = await VaultService.read(handle: handle, name: 'uncompressed_segment.txt'); - + final resultA = await VaultService.read( + handle: handle, + name: 'zstd_segment.txt', + ); + final resultB = await VaultService.read( + handle: handle, + name: 'brotli_segment.txt', + ); + final resultC = await VaultService.read( + handle: handle, + name: 'uncompressed_segment.txt', + ); + // Verify all three match original data - expect(resultA, dataA, reason: 'Zstd segment should decompress correctly'); - expect(resultB, dataB, reason: 'Brotli segment should decompress correctly'); - expect(resultC, dataC, reason: 'Uncompressed segment should read correctly'); - + expect( + resultA, + dataA, + reason: 'Zstd segment should decompress correctly', + ); + expect( + resultB, + dataB, + reason: 'Brotli segment should decompress correctly', + ); + expect( + resultC, + dataC, + reason: 'Uncompressed segment should read correctly', + ); + // Verify all three segments are in the list final names = await VaultService.list(handle: handle); expect(names.length, 3); - expect(names, containsAll(['zstd_segment.txt', 'brotli_segment.txt', 'uncompressed_segment.txt'])); - + expect( + names, + containsAll([ + 'zstd_segment.txt', + 'brotli_segment.txt', + 'uncompressed_segment.txt', + ]), + ); + await VaultService.close(handle: handle); }); @@ -511,7 +539,7 @@ void main() { test('crash recovery via WAL', () async { final path = '${tempDir.path}/wal.vault'; final key = await generateAes256GcmKey(); - + // Create vault final handle = await VaultService.create( path: path, @@ -519,36 +547,53 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 2 * 1024 * 1024, ); - + // Write some data final data1 = Uint8List.fromList([1, 2, 3, 4, 5]); - await VaultService.write(handle: handle, name: 'initial.bin', data: data1); - + await VaultService.write( + handle: handle, + name: 'initial.bin', + data: data1, + ); + // Close properly (commits WAL) await VaultService.close(handle: handle); - + // Reopen (WAL recovery runs but finds everything committed) final reopened = await VaultService.open(path: path, key: key); - + // Verify data survived - final result = await VaultService.read(handle: reopened, name: 'initial.bin'); + final result = await VaultService.read( + handle: reopened, + name: 'initial.bin', + ); expect(result, data1); - + // Write more data after recovery final data2 = Uint8List.fromList([6, 7, 8, 9]); - await VaultService.write(handle: reopened, name: 'after_recovery.bin', data: data2); - + await VaultService.write( + handle: reopened, + name: 'after_recovery.bin', + data: data2, + ); + // Close and reopen again await VaultService.close(handle: reopened); final reopened2 = await VaultService.open(path: path, key: key); - + // Both segments should exist - final result1 = await VaultService.read(handle: reopened2, name: 'initial.bin'); - final result2 = await VaultService.read(handle: reopened2, name: 'after_recovery.bin'); - + final result1 = await VaultService.read( + handle: reopened2, + name: 'initial.bin', + ); + final result2 = await VaultService.read( + handle: reopened2, + name: 'after_recovery.bin', + ); + expect(result1, data1); expect(result2, data2); - + await VaultService.close(handle: reopened2); }); // -- Defragmentation ------------------------------------------------ @@ -566,7 +611,9 @@ void main() { // Write A, B, C final dataA = Uint8List.fromList(List.generate(1000, (i) => i % 256)); - final dataC = Uint8List.fromList(List.generate(500, (i) => (i * 3) % 256)); + final dataC = Uint8List.fromList( + List.generate(500, (i) => (i * 3) % 256), + ); await VaultService.write(handle: handle, name: 'a.txt', data: dataA); await VaultService.write( handle: handle, @@ -631,10 +678,7 @@ void main() { ); // Grow to 1MB - await VaultService.resize( - handle: handle, - newCapacityBytes: 1024 * 1024, - ); + await VaultService.resize(handle: handle, newCapacityBytes: 1024 * 1024); final health = await VaultService.health(handle: handle); expect(health.totalBytes, BigInt.from(1024 * 1024)); @@ -668,10 +712,7 @@ void main() { await VaultService.defragment(handle: handle); // Shrink to 512KB (data fits) - await VaultService.resize( - handle: handle, - newCapacityBytes: 512 * 1024, - ); + await VaultService.resize(handle: handle, newCapacityBytes: 512 * 1024); final health = await VaultService.health(handle: handle); expect(health.totalBytes, BigInt.from(512 * 1024)); @@ -783,7 +824,7 @@ void main() { test('corrupted primary index falls back to shadow', () async { final path = '${tempDir.path}/shadow.vault'; final key = await generateAes256GcmKey(); - + // Create vault and write data final handle = await VaultService.create( path: path, @@ -791,52 +832,322 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 2 * 1024 * 1024, ); - + final originalData = Uint8List.fromList([1, 2, 3, 4, 5]); - await VaultService.write(handle: handle, name: 'important.bin', data: originalData); - + await VaultService.write( + handle: handle, + name: 'important.bin', + data: originalData, + ); + await VaultService.close(handle: handle); - + // At this point: // - Primary index has 1 segment // - Shadow index has 1 segment (backup) - + // Manually corrupt the PRIMARY index final file = File(path); final bytes = await file.readAsBytes(); - + // Primary index starts at byte 32 (after header) // Shadow index starts at byte 32 + 64KB final primaryIndexStart = 32; //final primaryIndexEnd = primaryIndexStart + (64 * 1024); - + // Corrupt some bytes in the primary index region // (but leave shadow index intact) for (int i = primaryIndexStart; i < primaryIndexStart + 100; i++) { bytes[i] = 0xFF; // Corrupt } - + await file.writeAsBytes(bytes); - + // Try to reopen // Rust should: // 1. Try to decrypt primary index → fail (corrupted) // 2. Fall back to shadow index → succeed // 3. Restore primary index from shadow final reopened = await VaultService.open(path: path, key: key); - + // Read data - should work because shadow index has it - final result = await VaultService.read(handle: reopened, name: 'important.bin'); - expect(result, originalData, reason: 'Shadow index should have preserved the segment'); - + final result = await VaultService.read( + handle: reopened, + name: 'important.bin', + ); + expect( + result, + originalData, + reason: 'Shadow index should have preserved the segment', + ); + await VaultService.close(handle: reopened); - + // Reopen again - primary should be restored now final reopened2 = await VaultService.open(path: path, key: key); - final result2 = await VaultService.read(handle: reopened2, name: 'important.bin'); + final result2 = await VaultService.read( + handle: reopened2, + name: 'important.bin', + ); expect(result2, originalData); - + await VaultService.close(handle: reopened2); }); + + // -- Streaming (writeStream / readStream) ---------------------------------- + + group('Streaming', () { + test('stream-write 10 MB then stream-read back byte-identical', () async { + final path = '${tempDir.path}/stream_10mb.vault'; + final key = await generateAes256GcmKey(); + const dataSize = 10 * 1024 * 1024; // 10 MB + const chunkSize = 64 * 1024; // 64 KB + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: + 16 * 1024 * 1024, // 16 MB — enough for encrypted overhead + ); + + // Build source data in memory for the round-trip assertion. + final sourceData = Uint8List(dataSize); + for (int i = 0; i < dataSize; i++) { + sourceData[i] = i % 256; + } + + // Emit source data in 64 KB chunks. + Stream chunks() async* { + int offset = 0; + while (offset < dataSize) { + final end = (offset + chunkSize).clamp(0, dataSize); + yield sourceData.sublist(offset, end); + offset = end; + } + } + + await VaultService.writeStream( + handle: handle, + name: 'large.bin', + totalSize: dataSize, + data: chunks(), + ); + + // Stream-read and reassemble. + final builder = BytesBuilder(copy: false); + await for (final chunk in VaultService.readStream( + handle: handle, + name: 'large.bin', + )) { + builder.add(chunk); + } + + expect(builder.takeBytes(), sourceData); + await VaultService.close(handle: handle); + }); + + test('stream-write then one-shot read (interop)', () async { + final path = '${tempDir.path}/stream_to_oneshot.vault'; + final key = await generateAes256GcmKey(); + const dataSize = 256 * 1024; // 256 KB + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: 2 * 1024 * 1024, + ); + + final sourceData = Uint8List.fromList( + List.generate(dataSize, (i) => (i * 3) % 256), + ); + + await VaultService.writeStream( + handle: handle, + name: 'interop.bin', + totalSize: dataSize, + data: Stream.fromIterable([sourceData]), + ); + + // VaultService.read() must be able to read a streaming segment. + final result = await VaultService.read( + handle: handle, + name: 'interop.bin', + ); + expect(result, sourceData); + + await VaultService.close(handle: handle); + }); + + test('one-shot write then stream-read (interop)', () async { + final path = '${tempDir.path}/oneshot_to_stream.vault'; + final key = await generateAes256GcmKey(); + const dataSize = 128 * 1024; // 128 KB + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: 2 * 1024 * 1024, + ); + + final sourceData = Uint8List.fromList( + List.generate(dataSize, (i) => (i * 7) % 256), + ); + + // VaultService.write() writes a monolithic segment. + await VaultService.write( + handle: handle, + name: 'interop2.bin', + data: sourceData, + ); + + // VaultService.readStream() must handle monolithic segments + // (Rust falls back to a one-shot read and emits one chunk). + final builder = BytesBuilder(copy: false); + await for (final chunk in VaultService.readStream( + handle: handle, + name: 'interop2.bin', + )) { + builder.add(chunk); + } + + expect(builder.takeBytes(), sourceData); + await VaultService.close(handle: handle); + }); + + test( + 'readStream emits multiple chunks for large streaming segment', + () async { + // Verifies that data actually arrives incrementally, not as one blob. + final path = '${tempDir.path}/progress.vault'; + final key = await generateAes256GcmKey(); + const dataSize = 512 * 1024; // 512 KB = 8 × 64 KB chunks + const chunkSize = 64 * 1024; + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: 4 * 1024 * 1024, + ); + + await VaultService.writeStream( + handle: handle, + name: 'chunked.bin', + totalSize: dataSize, + data: Stream.fromIterable( + List.generate(dataSize ~/ chunkSize, (_) => Uint8List(chunkSize)), + ), + ); + + final chunkLengths = []; + await for (final chunk in VaultService.readStream( + handle: handle, + name: 'chunked.bin', + )) { + chunkLengths.add(chunk.length); + } + + // Must have received more than one chunk. + expect(chunkLengths.length, greaterThan(1)); + // Total byte count must be exact. + expect(chunkLengths.fold(0, (a, b) => a + b), dataSize); + + await VaultService.close(handle: handle); + }, + ); + + test('stream-write with wrong totalSize throws ArgumentError', () async { + final path = '${tempDir.path}/wrong_size.vault'; + final key = await generateAes256GcmKey(); + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: 2 * 1024 * 1024, + ); + + // Underflow: stream emits fewer bytes than totalSize claims. + await expectLater( + VaultService.writeStream( + handle: handle, + name: 'underflow.bin', + totalSize: 100, // claims 100 bytes + data: Stream.fromIterable([ + Uint8List.fromList([1, 2, 3]), + ]), // only 3 bytes + ), + throwsA(isA()), + ); + + // Overflow: stream emits more bytes than totalSize allows. + await expectLater( + VaultService.writeStream( + handle: handle, + name: 'overflow.bin', + totalSize: 2, // claims only 2 bytes + data: Stream.fromIterable([Uint8List(1024)]), // 1 KB + ), + throwsA(isA()), + ); + + await VaultService.close(handle: handle); + }); + + test('50 MB stream-write and stream-read stays memory-bounded', () async { + // If memory were not bounded this would OOM on constrained devices. + // Successful completion without process termination is the assertion. + final path = '${tempDir.path}/large_50mb.vault'; + final key = await generateAes256GcmKey(); + const dataSize = 50 * 1024 * 1024; // 50 MB + const chunkSize = 64 * 1024; // 64 KB + + final handle = await VaultService.create( + path: path, + key: key, + algorithm: 'aes-256-gcm', + capacityBytes: 55 * 1024 * 1024, + ); + + // Generate stream lazily — never holds more than one 64 KB chunk in + // Dart memory. + int writeCounter = 0; + Stream lazyStream() async* { + int remaining = dataSize; + while (remaining > 0) { + final size = remaining < chunkSize ? remaining : chunkSize; + final chunk = Uint8List(size); + for (int i = 0; i < size; i++) { + chunk[i] = (writeCounter + i) % 256; + } + writeCounter += size; + remaining -= size; + yield chunk; + } + } + + await VaultService.writeStream( + handle: handle, + name: 'huge.bin', + totalSize: dataSize, + data: lazyStream(), + ); + + // Stream-read and count bytes without accumulating all data in memory. + int totalRead = 0; + await for (final chunk in VaultService.readStream( + handle: handle, + name: 'huge.bin', + )) { + totalRead += chunk.length; + } + + expect(totalRead, dataSize); + await VaultService.close(handle: handle); + }); + }); }); } diff --git a/integration_test/evfs_test.dart b/integration_test/evfs_test.dart index 1fd8005..ebd4521 100644 --- a/integration_test/evfs_test.dart +++ b/integration_test/evfs_test.dart @@ -415,12 +415,12 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 5 * 1024 * 1024, // 5MB ); - + // Prepare test data (compressible - lots of repeats) final dataA = Uint8List.fromList(List.filled(5000, 42)); final dataB = Uint8List.fromList(List.filled(5000, 99)); final dataC = Uint8List.fromList(List.filled(5000, 123)); - + // Write segment A with Zstd await VaultService.write( handle: handle, @@ -431,7 +431,7 @@ void main() { level: 3, ), ); - + // Write segment B with Brotli await VaultService.write( handle: handle, @@ -442,7 +442,7 @@ void main() { level: 4, ), ); - + // Write segment C with None (no compression) await VaultService.write( handle: handle, @@ -452,22 +452,50 @@ void main() { algorithm: CompressionAlgorithm.none, ), ); - + // Read all three back - final resultA = await VaultService.read(handle: handle, name: 'zstd_segment.txt'); - final resultB = await VaultService.read(handle: handle, name: 'brotli_segment.txt'); - final resultC = await VaultService.read(handle: handle, name: 'uncompressed_segment.txt'); - + final resultA = await VaultService.read( + handle: handle, + name: 'zstd_segment.txt', + ); + final resultB = await VaultService.read( + handle: handle, + name: 'brotli_segment.txt', + ); + final resultC = await VaultService.read( + handle: handle, + name: 'uncompressed_segment.txt', + ); + // Verify all three match original data - expect(resultA, dataA, reason: 'Zstd segment should decompress correctly'); - expect(resultB, dataB, reason: 'Brotli segment should decompress correctly'); - expect(resultC, dataC, reason: 'Uncompressed segment should read correctly'); - + expect( + resultA, + dataA, + reason: 'Zstd segment should decompress correctly', + ); + expect( + resultB, + dataB, + reason: 'Brotli segment should decompress correctly', + ); + expect( + resultC, + dataC, + reason: 'Uncompressed segment should read correctly', + ); + // Verify all three segments are in the list final names = await VaultService.list(handle: handle); expect(names.length, 3); - expect(names, containsAll(['zstd_segment.txt', 'brotli_segment.txt', 'uncompressed_segment.txt'])); - + expect( + names, + containsAll([ + 'zstd_segment.txt', + 'brotli_segment.txt', + 'uncompressed_segment.txt', + ]), + ); + await VaultService.close(handle: handle); }); @@ -511,7 +539,7 @@ void main() { test('crash recovery via WAL', () async { final path = '${tempDir.path}/wal.vault'; final key = await generateAes256GcmKey(); - + // Create vault final handle = await VaultService.create( path: path, @@ -519,42 +547,59 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 2 * 1024 * 1024, ); - + // Write some data final data1 = Uint8List.fromList([1, 2, 3, 4, 5]); - await VaultService.write(handle: handle, name: 'initial.bin', data: data1); - + await VaultService.write( + handle: handle, + name: 'initial.bin', + data: data1, + ); + // Close properly (commits WAL) await VaultService.close(handle: handle); - + // Reopen (WAL recovery runs but finds everything committed) final reopened = await VaultService.open(path: path, key: key); - + // Verify data survived - final result = await VaultService.read(handle: reopened, name: 'initial.bin'); + final result = await VaultService.read( + handle: reopened, + name: 'initial.bin', + ); expect(result, data1); - + // Write more data after recovery final data2 = Uint8List.fromList([6, 7, 8, 9]); - await VaultService.write(handle: reopened, name: 'after_recovery.bin', data: data2); - + await VaultService.write( + handle: reopened, + name: 'after_recovery.bin', + data: data2, + ); + // Close and reopen again await VaultService.close(handle: reopened); final reopened2 = await VaultService.open(path: path, key: key); - + // Both segments should exist - final result1 = await VaultService.read(handle: reopened2, name: 'initial.bin'); - final result2 = await VaultService.read(handle: reopened2, name: 'after_recovery.bin'); - + final result1 = await VaultService.read( + handle: reopened2, + name: 'initial.bin', + ); + final result2 = await VaultService.read( + handle: reopened2, + name: 'after_recovery.bin', + ); + expect(result1, data1); expect(result2, data2); - + await VaultService.close(handle: reopened2); }); test('corrupted primary index falls back to shadow', () async { final path = '${tempDir.path}/shadow.vault'; final key = await generateAes256GcmKey(); - + // Create vault and write data final handle = await VaultService.create( path: path, @@ -562,47 +607,60 @@ void main() { algorithm: 'aes-256-gcm', capacityBytes: 2 * 1024 * 1024, ); - + final originalData = Uint8List.fromList([1, 2, 3, 4, 5]); - await VaultService.write(handle: handle, name: 'important.bin', data: originalData); - + await VaultService.write( + handle: handle, + name: 'important.bin', + data: originalData, + ); + await VaultService.close(handle: handle); - // primary index has 1 segment // shadow index has 1 segment (backup) - + // Manually corrupt the PRIMARY index final file = File(path); final bytes = await file.readAsBytes(); - + // Primary index starts at byte 32 (after header) // Shadow index starts at byte 32 + 64KB final primaryIndexStart = 32; //final primaryIndexEnd = primaryIndexStart + (64 * 1024); - + // Corrupt some bytes in the primary index region // (but leave shadow index intact) for (int i = primaryIndexStart; i < primaryIndexStart + 100; i++) { bytes[i] = 0xFF; // Corrupt } - + await file.writeAsBytes(bytes); - + // Try to reopen final reopened = await VaultService.open(path: path, key: key); - + // Read data - should work because shadow index has it - final result = await VaultService.read(handle: reopened, name: 'important.bin'); - expect(result, originalData, reason: 'Shadow index should have preserved the segment'); - + final result = await VaultService.read( + handle: reopened, + name: 'important.bin', + ); + expect( + result, + originalData, + reason: 'Shadow index should have preserved the segment', + ); + await VaultService.close(handle: reopened); - + // Reopen again - primary should be restored now final reopened2 = await VaultService.open(path: path, key: key); - final result2 = await VaultService.read(handle: reopened2, name: 'important.bin'); + final result2 = await VaultService.read( + handle: reopened2, + name: 'important.bin', + ); expect(result2, originalData); - + await VaultService.close(handle: reopened2); }); }); From ab2ff6cba8321c8fb1f1ce661c966f0a51f8ad58 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Thu, 26 Mar 2026 13:09:32 +0100 Subject: [PATCH 18/21] feat(vault_service): expose onProgress callback in writeStream/readStream --- example/integration_test/evfs_test.dart | 30 ++++++++++++++++++++----- lib/src/evfs/vault_service.dart | 26 +++++++++++++++------ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/example/integration_test/evfs_test.dart b/example/integration_test/evfs_test.dart index fc6e3ad..eb6d778 100644 --- a/example/integration_test/evfs_test.dart +++ b/example/integration_test/evfs_test.dart @@ -1018,9 +1018,8 @@ void main() { }); test( - 'readStream emits multiple chunks for large streaming segment', + 'progress reporting during stream-write and stream-read', () async { - // Verifies that data actually arrives incrementally, not as one blob. final path = '${tempDir.path}/progress.vault'; final key = await generateAes256GcmKey(); const dataSize = 512 * 1024; // 512 KB = 8 × 64 KB chunks @@ -1033,26 +1032,45 @@ void main() { capacityBytes: 4 * 1024 * 1024, ); + // Track write progress. + final writeProgress = []; await VaultService.writeStream( handle: handle, - name: 'chunked.bin', + name: 'progress.bin', totalSize: dataSize, data: Stream.fromIterable( List.generate(dataSize ~/ chunkSize, (_) => Uint8List(chunkSize)), ), + onProgress: writeProgress.add, ); + // Write progress: monotonically increasing, ending at 1.0. + expect(writeProgress, isNotEmpty); + expect(writeProgress.last, 1.0); + for (int i = 1; i < writeProgress.length; i++) { + expect(writeProgress[i], greaterThanOrEqualTo(writeProgress[i - 1])); + } + + // Track read progress. + final readProgress = []; final chunkLengths = []; await for (final chunk in VaultService.readStream( handle: handle, - name: 'chunked.bin', + name: 'progress.bin', + onProgress: readProgress.add, )) { chunkLengths.add(chunk.length); } - // Must have received more than one chunk. + // Read progress: monotonically increasing, ending at 1.0. + expect(readProgress, isNotEmpty); + expect(readProgress.last, 1.0); + for (int i = 1; i < readProgress.length; i++) { + expect(readProgress[i], greaterThanOrEqualTo(readProgress[i - 1])); + } + + // Data arrives as multiple chunks with correct total. expect(chunkLengths.length, greaterThan(1)); - // Total byte count must be exact. expect(chunkLengths.fold(0, (a, b) => a + b), dataSize); await VaultService.close(handle: handle); diff --git a/lib/src/evfs/vault_service.dart b/lib/src/evfs/vault_service.dart index 2e449a9..e98f1e3 100644 --- a/lib/src/evfs/vault_service.dart +++ b/lib/src/evfs/vault_service.dart @@ -64,11 +64,14 @@ class VaultService { /// Pipes [data] through a temporary file on disk so that Dart RAM usage /// is bounded to a single chunk. [totalSize] must equal the exact number /// of bytes that [data] will emit. + /// + /// [onProgress] is called with values in (0.0, 1.0] as chunks are encrypted. static Future writeStream({ required rust_types.VaultHandle handle, required String name, required int totalSize, required Stream data, + void Function(double progress)? onProgress, }) async { final tempDir = await Directory.systemTemp.createTemp('vault_write_stream'); final tempFile = File('${tempDir.path}/payload.bin'); @@ -103,13 +106,21 @@ class VaultService { // Delegate to vaultWriteFile which reads the temp file in 64 KB chunks // inside Rust — keeping the end-to-end memory footprint bounded. - await _guardedStream( + final progressStream = _guardedStream( () => rust_evfs.vaultWriteFile( handle: handle, name: name, filePath: tempFile.path, ), - ).drain(); + ); + + if (onProgress != null) { + await for (final p in progressStream) { + onProgress(p); + } + } else { + await progressStream.drain(); + } } finally { await tempDir.delete(recursive: true); } @@ -123,11 +134,13 @@ class VaultService { return rust_evfs.vaultRead(handle: handle, name: name); } - /// Read a named segment. Decompression bytes are sent chunk-by-chunk as they are processed by - /// the vault. + /// Read a named segment as a stream of decrypted chunks. + /// + /// [onProgress] is called with values in (0.0, 1.0] as chunks are decrypted. static Stream readStream({ required rust_types.VaultHandle handle, required String name, + void Function(double progress)? onProgress, }) { final controller = StreamController(); final dataSink = RustStreamSink(); @@ -156,9 +169,8 @@ class VaultService { cancelOnError: true, ); - // Drain the progress sink silently — we expose data chunks, not - // progress, through the public Stream API. - progressSink.stream.listen(null); + // Forward progress to caller if provided, otherwise drain silently. + progressSink.stream.listen(onProgress); // Kick off the Rust streaming read. Errors from Rust arrive via // the returned Future (executeNormal, not unawaited), so we From f721568501f3bd73f936af95cf5c8ddaa64d23ac Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Thu, 26 Mar 2026 13:29:50 +0100 Subject: [PATCH 19/21] fix(evfs): harden streaming vault I/O and resource safety --- lib/src/evfs/vault_service.dart | 21 +++++++++++++++++---- rust/src/api/evfs/helpers.rs | 7 ++++--- rust/src/api/evfs/mod.rs | 29 +++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/lib/src/evfs/vault_service.dart b/lib/src/evfs/vault_service.dart index e98f1e3..5537beb 100644 --- a/lib/src/evfs/vault_service.dart +++ b/lib/src/evfs/vault_service.dart @@ -84,11 +84,14 @@ class VaultService { try { await for (final chunk in data) { + final prevBytes = bytesReceived; bytesReceived += chunk.length; if (bytesReceived > totalSize) { throw ArgumentError( 'writeStream: stream overflow — ' - 'received >$bytesReceived bytes but totalSize is $totalSize', + 'received $bytesReceived bytes ' + '(chunk of ${chunk.length} at offset $prevBytes) ' + 'but totalSize is $totalSize', ); } await raf.writeFrom(chunk); @@ -142,14 +145,22 @@ class VaultService { required String name, void Function(double progress)? onProgress, }) { - final controller = StreamController(); final dataSink = RustStreamSink(); final progressSink = RustStreamSink(); + StreamSubscription? dataSub; + StreamSubscription? progressSub; + + final controller = StreamController( + onCancel: () { + dataSub?.cancel(); + progressSub?.cancel(); + }, + ); runZonedGuarded( () { // Forward each plaintext chunk from Rust to the public stream. - dataSink.stream.listen( + dataSub = dataSink.stream.listen( controller.add, onError: (Object e, StackTrace s) { if (!controller.isClosed) { @@ -170,7 +181,9 @@ class VaultService { ); // Forward progress to caller if provided, otherwise drain silently. - progressSink.stream.listen(onProgress); + if (onProgress != null) { + progressSub = progressSink.stream.listen(onProgress); + } // Kick off the Rust streaming read. Errors from Rust arrive via // the returned Future (executeNormal, not unawaited), so we diff --git a/rust/src/api/evfs/helpers.rs b/rust/src/api/evfs/helpers.rs index 0d8f574..7621725 100644 --- a/rust/src/api/evfs/helpers.rs +++ b/rust/src/api/evfs/helpers.rs @@ -123,9 +123,10 @@ pub(crate) fn decrypt_streaming_chunks( let mut decomp_buf = Vec::with_capacity(crate::core::streaming::CHUNK_SIZE * 2); for i in 0..chunk_count { - let chunk_offset = data_region - + seg_offset - + (i as u64 * crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64); + let chunk_offset = (i as u64) + .checked_mul(crate::core::streaming::ENCRYPTED_CHUNK_SIZE as u64) + .and_then(|co| data_region.checked_add(seg_offset)?.checked_add(co)) + .ok_or_else(|| CryptoError::InvalidParameter("chunk offset overflow".into()))?; file.seek(SeekFrom::Start(chunk_offset))?; let mut encrypted = vec![0u8; crate::core::streaming::ENCRYPTED_CHUNK_SIZE]; diff --git a/rust/src/api/evfs/mod.rs b/rust/src/api/evfs/mod.rs index b951210..e803957 100644 --- a/rust/src/api/evfs/mod.rs +++ b/rust/src/api/evfs/mod.rs @@ -506,7 +506,9 @@ pub fn vault_write_stream( for mut input in data_stream { hasher.update(&input); - total_received += input.len() as u64; + total_received = total_received + .checked_add(input.len() as u64) + .ok_or_else(|| CryptoError::InvalidParameter("stream size overflow".into()))?; if total_received > total_plaintext_size { input.zeroize(); @@ -569,6 +571,9 @@ pub fn vault_write_stream( padded.zeroize(); final_result?; + // Single durability barrier after all chunks — WAL provides atomicity + handle.file.sync_all()?; + let actual_chunks = chunk_index + 1; if actual_chunks != expected_chunks as u64 { return Err(CryptoError::VaultCorrupted(format!( @@ -599,6 +604,7 @@ pub fn vault_write_stream( )?; handle.wal.commit()?; + handle.wal.checkpoint()?; Ok(()) } @@ -623,7 +629,11 @@ pub fn vault_write_file( file.seek(SeekFrom::Start(0))?; let mut bytes_read: u64 = 0; + let mut read_error: Option = None; let data_stream = std::iter::from_fn(|| { + if read_error.is_some() { + return None; + } let mut buf = vec![0u8; CHUNK_SIZE]; match file.read(&mut buf) { Ok(0) => None, @@ -635,11 +645,23 @@ pub fn vault_write_file( } Some(buf) } - Err(_) => None, + Err(e) => { + read_error = Some(e); + None + } } }); - vault_write_stream(handle, name, file_size, data_stream)?; + let result = vault_write_stream(handle, name, file_size, data_stream); + + // Surface the real I/O error instead of a misleading "stream underflow" + if let Some(io_err) = read_error { + return Err(CryptoError::IoError(format!( + "read error on '{file_path}': {io_err}" + ))); + } + result?; + let _ = on_progress.add(1.0); Ok(()) } @@ -689,7 +711,6 @@ fn write_encrypted_chunk( file.seek(SeekFrom::Start(abs_offset))?; file.write_all(&wire)?; - file.sync_all()?; Ok(()) } From 1b627b7dc8412d71de82265ddee68ca0f5457a6a Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Thu, 26 Mar 2026 13:37:50 +0100 Subject: [PATCH 20/21] feat(example): add streaming I/O section to vault tab --- example/lib/main.dart | 132 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 2 deletions(-) diff --git a/example/lib/main.dart b/example/lib/main.dart index 6af7ae8..7ac6956 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -21,7 +21,7 @@ class ExampleApp extends StatelessWidget { @override Widget build(BuildContext context) { return MaterialApp( - title: 'M-Security v0.3.0', + title: 'M-Security v0.3.2', theme: ThemeData(colorSchemeSeed: Colors.blue, useMaterial3: true), home: const DemoHome(), ); @@ -41,7 +41,7 @@ class _DemoHomeState extends State { @override Widget build(BuildContext context) { return Scaffold( - appBar: AppBar(title: const Text('M-Security v0.3.0')), + appBar: AppBar(title: const Text('M-Security v0.3.2')), body: IndexedStack( index: _tab, children: const [ @@ -605,10 +605,12 @@ class _VaultTabState extends State<_VaultTab> { final _vaultSizeMb = TextEditingController(text: '5'); final _segName = TextEditingController(text: 'secret.txt'); final _segData = TextEditingController(text: 'Vault data here'); + final _streamSizeKb = TextEditingController(text: '512'); String _status = ''; List _segments = []; String _readResult = ''; String _capacityInfo = ''; + double _streamProgress = 0; bool _loading = false; bool _vaultOpen = false; String _compAlgo = 'Zstd'; @@ -729,6 +731,90 @@ class _VaultTabState extends State<_VaultTab> { setState(() => _loading = false); } + Future _streamWrite() async { + if (!_vaultOpen || _handle == null) return; + setState(() { + _loading = true; + _streamProgress = 0; + }); + try { + final sizeBytes = int.parse(_streamSizeKb.text) * 1024; + const chunkSize = 64 * 1024; + + // Generate data as a stream of 64KB chunks + Stream dataStream() async* { + var remaining = sizeBytes; + var offset = 0; + while (remaining > 0) { + final n = remaining < chunkSize ? remaining : chunkSize; + final chunk = Uint8List(n); + for (var i = 0; i < n; i++) { + chunk[i] = (offset + i) % 256; + } + yield chunk; + offset += n; + remaining -= n; + } + } + + await VaultService.writeStream( + handle: _handle!, + name: 'stream-${_streamSizeKb.text}kb.bin', + totalSize: sizeBytes, + data: dataStream(), + onProgress: (p) => setState(() => _streamProgress = p), + ); + + _status = 'Stream-wrote ${_fmtBytes(BigInt.from(sizeBytes))} as ' + '"stream-${_streamSizeKb.text}kb.bin"'; + await _refreshList(); + await _refreshCapacity(); + } catch (e) { + _status = 'Error: $e'; + } + setState(() => _loading = false); + } + + Future _streamRead() async { + if (!_vaultOpen || _handle == null) return; + final name = 'stream-${_streamSizeKb.text}kb.bin'; + if (!_segments.contains(name)) { + setState(() => _status = 'Write "$name" first'); + return; + } + setState(() { + _loading = true; + _streamProgress = 0; + }); + try { + final chunks = []; + await for (final chunk in VaultService.readStream( + handle: _handle!, + name: name, + onProgress: (p) => setState(() => _streamProgress = p), + )) { + chunks.add(chunk); + } + final total = chunks.fold(0, (sum, c) => sum + c.length); + + // Verify pattern + var offset = 0; + var match = true; + for (final chunk in chunks) { + for (var i = 0; i < chunk.length && match; i++) { + if (chunk[i] != (offset + i) % 256) match = false; + } + offset += chunk.length; + } + + _status = 'Stream-read "$name": ${_fmtBytes(BigInt.from(total))} ' + 'in ${chunks.length} chunks — ${match ? "PASS" : "FAIL"}'; + } catch (e) { + _status = 'Error: $e'; + } + setState(() => _loading = false); + } + Future _refreshList() async { if (!_vaultOpen || _handle == null) return; _segments = await VaultService.list(handle: _handle!); @@ -895,6 +981,48 @@ class _VaultTabState extends State<_VaultTab> { ), )), if (_readResult.isNotEmpty) _ResultCard('Read', _readResult), + + const Divider(height: 24), + + // Streaming segment I/O + Text('Streaming I/O', + style: Theme.of(context).textTheme.titleSmall), + const SizedBox(height: 8), + TextField( + controller: _streamSizeKb, + keyboardType: TextInputType.number, + decoration: const InputDecoration( + labelText: 'Stream size (KB)', + border: OutlineInputBorder(), + isDense: true, + ), + ), + const SizedBox(height: 8), + Row( + children: [ + Expanded( + child: FilledButton.icon( + onPressed: _loading ? null : _streamWrite, + icon: const Icon(Icons.upload, size: 18), + label: const Text('Stream Write'), + ), + ), + const SizedBox(width: 8), + Expanded( + child: FilledButton.tonalIcon( + onPressed: _loading ? null : _streamRead, + icon: const Icon(Icons.download, size: 18), + label: const Text('Stream Read'), + ), + ), + ], + ), + if (_streamProgress > 0) + Padding( + padding: const EdgeInsets.only(top: 8), + child: LinearProgressIndicator( + value: _loading ? _streamProgress : 1), + ), ], if (_loading) const _Loader(), From ef61e0b365be0fabef4bcd202a1e7d0d3f8eb6e4 Mon Sep 17 00:00:00 2001 From: Adel-Ayoub Date: Thu, 26 Mar 2026 13:37:58 +0100 Subject: [PATCH 21/21] docs(changelog): add 0.3.1 and 0.3.2 release notes --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f47a0..da632c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,48 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.2] - 2026-03-26 + +### Added + +- `vault_write_stream()` for constant-memory chunked segment writes — encrypts data in 64 KB chunks without loading the full segment into RAM. +- `vault_read_stream()` for chunked segment reads via `StreamSink` — delivers decrypted data as a stream of byte chunks. +- Per-chunk AEAD encryption with domain-separated nonce derivation (`0x01` prefix, chunk index, generation) — provably disjoint from monolithic nonce space. +- `VaultChunkAad` struct binding generation and chunk position to each chunk's authentication tag (cross-segment splice defense). +- `vault_write_file()` FRB-callable wrapper that reads a file in 64 KB chunks and pipes into `vault_write_stream`. +- Dart `VaultService.writeStream()` — accepts a `Stream`, buffers to a temp file, and delegates to Rust for bounded-memory encryption. +- Dart `VaultService.readStream()` — returns a `Stream` of decrypted chunks with optional `onProgress` callback. +- Streaming interop: segments written via streaming can be read one-shot (and vice versa). +- Integration tests for streaming: 10 MB roundtrip, write/read interop, progress reporting, error handling, and 50 MB memory-bounded validation. +- Example app streaming I/O section in the Vault tab — stream-write and stream-read with configurable size and live progress bar. + +### Fixed + +- Checked arithmetic in `decrypt_streaming_chunks` read path — prevents wrong-region reads from crafted chunk counts. +- I/O errors in `vault_write_file` now surface as `CryptoError::IoError` instead of a misleading "stream underflow" message. +- `total_received` accumulation uses `checked_add` to prevent silent overflow on pathological input. +- Per-chunk `fsync` replaced with a single post-loop durability barrier — reduces streaming write I/O from O(N) fsyncs to O(1). +- WAL checkpoint after streaming write commit — prevents unbounded WAL growth. +- `readStream` cancellation leak — `onCancel` handler now cleans up data and progress subscriptions. +- Overflow error message now reports exact byte count, chunk size, and offset for easier debugging. + +## [0.3.1] - 2026-03-16 + +### Added + +- Vault defragmentation: `vault_defragment()` compacts segments toward data region start, coalescing all free space with per-move WAL protection and post-commit secure erase. +- Vault resize: `vault_resize()` grows or shrinks vault capacity, relocating shadow index and WAL region. +- Vault health check: `vault_health()` returns `VaultHealthInfo` with fragmentation %, free region count, largest contiguous block, and consistency invariant. +- Dynamic index sizing: `compute_index_size(capacity)` scales segment index proportionally (64 KB per MB, min 64 KB, max 16 MB cap) — replaces fixed 64 KB `INDEX_PAD_SIZE`. +- Dart `VaultService.defragment()`, `VaultService.resize()`, and `VaultService.health()` wrappers with integration tests. +- Example app vault maintenance UI (defrag, resize, health). + +### Fixed + +- Nonce reuse prevention after WAL recovery by hardening defrag backup path. +- Resize and defrag crash recovery hardening (fsync ordering, OOM guard, bounds check, health invariant overflow-safe check). +- Segment index size now scales with vault capacity — fixes OOM on large vaults with fixed 64 KB index. +- pub.dev score improvements. ## 0.3.0 - 2026-03-07