diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0fac45..5d6bcbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: env: RUST_BACKTRACE: 1 CARGO_TERM_COLOR: always + RUSTFLAGS: "-Dwarnings" + RUSTDOCFLAGS: "-Dwarnings" jobs: ci-pass: @@ -145,4 +147,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Format check run: | + cargo doc cargo fmt -- --check + cargo clippy --all-targets diff --git a/Cargo.toml b/Cargo.toml index a5402bf..a47ed08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } getrandom = { version = "0.2", features = ["js"] } thiserror = "1" tracing = "0.1" -compact-encoding = "1" +compact-encoding = "2" flat-tree = "6" merkle-tree-stream = "0.12" pretty-hash = "0.4" @@ -47,8 +47,8 @@ random-access-disk = { version = "3", default-features = false } [dev-dependencies] anyhow = "1.0.70" -proptest = "1.1.0" -proptest-derive = "0.2.0" +proptest = "1.6.0" +proptest-derive = "0.5.1" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" diff --git a/src/common/mod.rs b/src/common/mod.rs index f5fb6ba..9df6430 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -16,7 +16,7 @@ pub use self::store::Store; pub(crate) use self::store::{StoreInfo, StoreInfoInstruction, StoreInfoType}; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct BitfieldUpdate { +pub(crate) struct BitfieldUpdate { pub(crate) drop: bool, pub(crate) start: u64, pub(crate) length: u64, diff --git a/src/common/node.rs b/src/common/node.rs index 1c78144..b152779 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -23,6 +23,8 @@ pub struct Node { /// This node's index in the Merkle tree pub(crate) index: u64, /// Hash of the data in this node + // TODO make this [u8; 32] like: + // https://github.com/holepunchto/hypercore/blob/d21ebdeca1b27eb4c2232f8af17d5ae939ee97f2/lib/messages.js#L246 pub(crate) hash: Vec, /// Number of bytes in this [`Node::data`] pub(crate) length: u64, diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index f744048..2653361 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -3,7 +3,7 @@ use blake2::{ Blake2b, Blake2bMac, Digest, }; use byteorder::{BigEndian, WriteBytesExt}; -use compact_encoding::State; +use compact_encoding::{as_array, to_encoded_bytes, EncodingError, FixedWidthEncoding}; use ed25519_dalek::VerifyingKey; use merkle_tree_stream::Node as NodeTrait; use std::convert::AsRef; @@ -124,10 +124,9 @@ impl Hash { /// Hash data pub(crate) fn data(data: &[u8]) -> Self { - let (mut state, mut size) = State::new_with_size(8); - state - .encode_u64(data.len() as u64, &mut size) - .expect("Encoding u64 should not fail"); + let size = + (|| Ok::<_, EncodingError>(to_encoded_bytes!((data.len() as u64).as_fixed_width())))() + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b256::new(); hasher.update(LEAF_TYPE); @@ -147,10 +146,10 @@ impl Hash { (right, left) }; - let (mut state, mut size) = State::new_with_size(8); - state - .encode_u64(node1.length + node2.length, &mut size) - .expect("Encoding u64 should not fail"); + let len = node1.length + node2.length; + let size: Box<[u8]> = + (|| Ok::<_, EncodingError>(to_encoded_bytes!(len.as_fixed_width())))() + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b256::new(); hasher.update(PARENT_TYPE); @@ -170,13 +169,13 @@ impl Hash { for node in roots { let node = node.as_ref(); - let (mut state, mut buffer) = State::new_with_size(16); - state - .encode_u64(node.index(), &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_u64(node.len(), &mut buffer) - .expect("Encoding u64 should not fail"); + let buffer = (|| { + Ok::<_, EncodingError>(to_encoded_bytes!( + node.index().as_fixed_width(), + node.len().as_fixed_width() + )) + })() + .expect("Encoding u64 should not fail"); hasher.update(node.hash()); hasher.update(&buffer[..8]); @@ -212,20 +211,15 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 pub(crate) fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { - let (mut state, mut buffer) = State::new_with_size(80); - state - .encode_fixed_32(&TREE, &mut buffer) - .expect("Should be able "); - state - .encode_fixed_32(hash, &mut buffer) - .expect("Encoding fixed 32 bytes should not fail"); - state - .encode_u64(length, &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_u64(fork, &mut buffer) - .expect("Encoding u64 should not fail"); - buffer + (|| { + Ok::<_, EncodingError>(to_encoded_bytes!( + &TREE, + as_array::<32>(hash)?, + length.as_fixed_width(), + fork.as_fixed_width() + )) + })() + .expect("Encoding should not fail") } #[cfg(test)] diff --git a/src/encoding.rs b/src/encoding.rs index ed049a6..b54fd05 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,289 +1,281 @@ //! Hypercore-specific compact encodings -pub use compact_encoding::{CompactEncoding, EncodingError, EncodingErrorKind, State}; -use std::convert::TryInto; -use std::ops::{Deref, DerefMut}; - use crate::{ crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; +use compact_encoding::{ + as_array, encode_bytes_fixed, encoded_size_usize, map_decode, map_encode, sum_encoded_size, + take_array, write_slice, CompactEncoding, EncodingError, EncodingErrorKind, VecEncodable, +}; -#[derive(Debug, Clone)] -/// Wrapper struct for compact_encoding::State -pub struct HypercoreState(pub State); - -impl Default for HypercoreState { - /// Passthrought to compact_encoding - fn default() -> Self { - Self::new() - } -} - -impl HypercoreState { - /// Passthrought to compact_encoding - pub fn new() -> HypercoreState { - HypercoreState(State::new()) - } - - /// Passthrought to compact_encoding - pub fn new_with_size(size: usize) -> (HypercoreState, Box<[u8]>) { - let (state, buffer) = State::new_with_size(size); - (HypercoreState(state), buffer) - } - - /// Passthrought to compact_encoding - pub fn new_with_start_and_end(start: usize, end: usize) -> HypercoreState { - HypercoreState(State::new_with_start_and_end(start, end)) +impl CompactEncoding for Node { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.index, self.length) + 32) } - /// Passthrought to compact_encoding - pub fn from_buffer(buffer: &[u8]) -> HypercoreState { - HypercoreState(State::from_buffer(buffer)) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let hash = as_array::<32>(&self.hash)?; + Ok(map_encode!(buffer, self.index, self.length, hash)) } -} - -impl Deref for HypercoreState { - type Target = State; - fn deref(&self) -> &Self::Target { - &self.0 + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((index, length, hash), rest) = map_decode!(buffer, [u64, u64, [u8; 32]]); + Ok((Node::new(index, hash.to_vec(), length), rest)) } } -impl DerefMut for HypercoreState { - fn deref_mut(&mut self) -> &mut State { - &mut self.0 +impl VecEncodable for Node { + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized, + { + let mut out = encoded_size_usize(vec.len()); + for x in vec { + out += x.encoded_size()?; + } + Ok(out) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Node) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.length)?; - self.0.preencode_fixed_32() +impl CompactEncoding for RequestBlock { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.index, self.nodes)) } - fn encode(&mut self, value: &Node, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.length, buffer)?; - self.0.encode_fixed_32(&value.hash, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.index, self.nodes)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let hash: Box<[u8]> = self.0.decode_fixed_32(buffer)?; - Ok(Node::new(index, hash.to_vec(), length)) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((index, nodes), rest) = map_decode!(buffer, [u64, u64]); + Ok((RequestBlock { index, nodes }, rest)) } } -impl CompactEncoding> for HypercoreState { - fn preencode(&mut self, value: &Vec) -> Result { - let len = value.len(); - self.0.preencode(&len)?; - for val in value { - self.preencode(val)?; - } - Ok(self.end()) +impl CompactEncoding for RequestSeek { + fn encoded_size(&self) -> Result { + self.bytes.encoded_size() } - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { - let len = value.len(); - self.0.encode(&len, buffer)?; - for val in value { - self.encode(val, buffer)?; - } - Ok(self.start()) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + self.bytes.encode(buffer) } - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len: usize = self.0.decode(buffer)?; - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode(buffer)?); - } - Ok(value) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (bytes, rest) = u64::decode(buffer)?; + Ok((RequestSeek { bytes }, rest)) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestBlock) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.nodes) +impl CompactEncoding for RequestUpgrade { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.start, self.length)) } - fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.nodes, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.start, self.length)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let nodes: u64 = self.0.decode(buffer)?; - Ok(RequestBlock { index, nodes }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((start, length), rest) = map_decode!(buffer, [u64, u64]); + Ok((RequestUpgrade { start, length }, rest)) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestSeek) -> Result { - self.0.preencode(&value.bytes) +impl CompactEncoding for DataBlock { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.index, self.value, self.nodes)) } - fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) -> Result { - self.0.encode(&value.bytes, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.index, self.value, self.nodes)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let bytes: u64 = self.0.decode(buffer)?; - Ok(RequestSeek { bytes }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((index, value, nodes), rest) = map_decode!(buffer, [u64, Vec, Vec]); + Ok(( + DataBlock { + index, + value, + nodes, + }, + rest, + )) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestUpgrade) -> Result { - self.0.preencode(&value.start)?; - self.0.preencode(&value.length) +impl CompactEncoding for DataHash { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.index, self.nodes)) } - fn encode( - &mut self, - value: &RequestUpgrade, - buffer: &mut [u8], - ) -> Result { - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.index, self.nodes)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - Ok(RequestUpgrade { start, length }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((index, nodes), rest) = map_decode!(buffer, [u64, Vec]); + Ok((DataHash { index, nodes }, rest)) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataBlock) -> Result { - self.0.preencode(&value.index)?; - self.0.preencode(&value.value)?; - self.preencode(&value.nodes) +impl CompactEncoding for DataSeek { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.bytes, self.nodes)) } - fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.0.encode(&value.value, buffer)?; - self.encode(&value.nodes, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.bytes, self.nodes)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let value: Vec = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataBlock { - index, - value, - nodes, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((bytes, nodes), rest) = map_decode!(buffer, [u64, Vec]); + Ok((DataSeek { bytes, nodes }, rest)) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataHash) -> Result { - self.0.preencode(&value.index)?; - self.preencode(&value.nodes) - } - - fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) -> Result { - self.0.encode(&value.index, buffer)?; - self.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let index: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataHash { index, nodes }) +// from: +// https://github.com/holepunchto/hypercore/blob/d21ebdeca1b27eb4c2232f8af17d5ae939ee97f2/lib/messages.js#L394 +impl CompactEncoding for DataUpgrade { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!( + self.start, + self.length, + self.nodes, + self.additional_nodes, + self.signature + )) + } + + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!( + buffer, + self.start, + self.length, + self.nodes, + self.additional_nodes, + self.signature + )) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((start, length, nodes, additional_nodes, signature), rest) = + map_decode!(buffer, [u64, u64, Vec, Vec, Vec]); + Ok(( + DataUpgrade { + start, + length, + nodes, + additional_nodes, + signature, + }, + rest, + )) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataSeek) -> Result { - self.0.preencode(&value.bytes)?; - self.preencode(&value.nodes) - } - - fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) -> Result { - self.0.encode(&value.bytes, buffer)?; - self.encode(&value.nodes, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let bytes: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - Ok(DataSeek { bytes, nodes }) +impl CompactEncoding for ManifestSigner { + fn encoded_size(&self) -> Result { + Ok( + 1 /* Signature */ + 32 /* namespace */ + 32, /* public_key */ + ) } -} -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataUpgrade) -> Result { - self.0.preencode(&value.start)?; - self.0.preencode(&value.length)?; - self.preencode(&value.nodes)?; - self.preencode(&value.additional_nodes)?; - self.0.preencode(&value.signature) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = if &self.signature == "ed25519" { + write_slice(&[0], buffer)? + } else { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature type: {}", &self.signature), + )); + }; + let rest = encode_bytes_fixed(&self.namespace, rest)?; + encode_bytes_fixed(&self.public_key, rest) } - fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) -> Result { - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer)?; - self.encode(&value.nodes, buffer)?; - self.encode(&value.additional_nodes, buffer)?; - self.0.encode(&value.signature, buffer) - } + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([signature_id], rest) = take_array::<1>(buffer)?; + let signature: String = if signature_id != 0 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature id: {signature_id}"), + )); + } else { + "ed25519".to_string() + }; - fn decode(&mut self, buffer: &[u8]) -> Result { - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let nodes: Vec = self.decode(buffer)?; - let additional_nodes: Vec = self.decode(buffer)?; - let signature: Vec = self.0.decode(buffer)?; - Ok(DataUpgrade { - start, - length, - nodes, - additional_nodes, - signature, - }) + let (namespace, rest) = take_array::<32>(rest)?; + let (public_key, rest) = take_array::<32>(rest)?; + Ok(( + ManifestSigner { + signature, + namespace, + public_key, + }, + rest, + )) } } -impl CompactEncoding for State { - fn preencode(&mut self, value: &Manifest) -> Result { - self.add_end(1)?; // Version - self.add_end(1)?; // hash in one byte - self.add_end(1)?; // type in one byte - self.preencode(&value.signer) +impl CompactEncoding for Manifest { + fn encoded_size(&self) -> Result { + Ok(1 // Version + + 1 // hash in one byte + + 1 // type in one byte + + self.signer.encoded_size()?) } - fn encode(&mut self, value: &Manifest, buffer: &mut [u8]) -> Result { - self.set_byte_to_buffer(0, buffer)?; // Version - if &value.hash == "blake2b" { - self.set_byte_to_buffer(0, buffer)?; // Version + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = write_slice(&[0], buffer)?; + let rest = if &self.hash == "blake2b" { + write_slice(&[0], rest)? } else { return Err(EncodingError::new( EncodingErrorKind::InvalidData, - &format!("Unknown hash: {}", &value.hash), + &format!("Unknown hash: {}", &self.hash), )); - } - // Type. 0: static, 1: signer, 2: multiple signers - self.set_byte_to_buffer(1, buffer)?; // Version - self.encode(&value.signer, buffer) + }; + let rest = write_slice(&[1], rest)?; + self.signer.encode(rest) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let version: u8 = self.decode_u8(buffer)?; + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([version], rest) = take_array::<1>(buffer)?; if version != 0 { panic!("Unknown manifest version {}", version); } - let hash_id: u8 = self.decode_u8(buffer)?; + let ([hash_id], rest) = take_array::<1>(rest)?; let hash: String = if hash_id != 0 { return Err(EncodingError::new( EncodingErrorKind::InvalidData, @@ -292,79 +284,14 @@ impl CompactEncoding for State { } else { "blake2b".to_string() }; - - let manifest_type: u8 = self.decode_u8(buffer)?; + let ([manifest_type], rest) = take_array::<1>(rest)?; if manifest_type != 1 { return Err(EncodingError::new( EncodingErrorKind::InvalidData, &format!("Unknown manifest type: {manifest_type}"), )); } - let signer: ManifestSigner = self.decode(buffer)?; - - Ok(Manifest { hash, signer }) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, _value: &ManifestSigner) -> Result { - self.add_end(1)?; // Signature - self.preencode_fixed_32()?; - self.preencode_fixed_32() - } - - fn encode( - &mut self, - value: &ManifestSigner, - buffer: &mut [u8], - ) -> Result { - if &value.signature == "ed25519" { - self.set_byte_to_buffer(0, buffer)?; - } else { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown signature type: {}", &value.signature), - )); - } - self.encode_fixed_32(&value.namespace, buffer)?; - self.encode_fixed_32(&value.public_key, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let signature_id: u8 = self.decode_u8(buffer)?; - let signature: String = if signature_id != 0 { - return Err(EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Unknown signature id: {signature_id}"), - )); - } else { - "ed25519".to_string() - }; - let namespace: [u8; 32] = - self.decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid namespace in manifest signer", - ) - })?; - let public_key: [u8; 32] = - self.decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid public key in manifest signer", - ) - })?; - - Ok(ManifestSigner { - signature, - namespace, - public_key, - }) + let (signer, rest) = ManifestSigner::decode(rest)?; + Ok((Manifest { hash, signer }, rest)) } } diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index e368100..c8eeac4 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,8 @@ -use crate::encoding::{CompactEncoding, EncodingError, HypercoreState}; +use compact_encoding::{ + map_decode, map_encode, sum_encoded_size, take_array, take_array_mut, write_array, + CompactEncoding, EncodingError, +}; + use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade @@ -10,72 +14,75 @@ pub(crate) struct EntryTreeUpgrade { pub(crate) signature: Box<[u8]>, } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &EntryTreeUpgrade) -> Result { - self.0.preencode(&value.fork)?; - self.0.preencode(&value.ancestors)?; - self.0.preencode(&value.length)?; - self.0.preencode(&value.signature) +impl CompactEncoding for EntryTreeUpgrade { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!( + self.fork, + self.ancestors, + self.length, + self.signature + )) } - fn encode( - &mut self, - value: &EntryTreeUpgrade, - buffer: &mut [u8], - ) -> Result { - self.0.encode(&value.fork, buffer)?; - self.0.encode(&value.ancestors, buffer)?; - self.0.encode(&value.length, buffer)?; - self.0.encode(&value.signature, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!( + buffer, + self.fork, + self.ancestors, + self.length, + self.signature + )) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let fork: u64 = self.0.decode(buffer)?; - let ancestors: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - let signature: Box<[u8]> = self.0.decode(buffer)?; - Ok(EntryTreeUpgrade { - fork, - ancestors, - length, - signature, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((fork, ancestors, length, signature), rest) = + map_decode!(buffer, [u64, u64, u64, Box<[u8]>]); + Ok(( + Self { + fork, + ancestors, + length, + signature, + }, + rest, + )) } } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &BitfieldUpdate) -> Result { - self.0.add_end(1)?; - self.0.preencode(&value.start)?; - self.0.preencode(&value.length) +impl CompactEncoding for BitfieldUpdate { + fn encoded_size(&self) -> Result { + Ok(1 + sum_encoded_size!(self.start, self.length)) } - fn encode( - &mut self, - value: &BitfieldUpdate, - buffer: &mut [u8], - ) -> Result { - let flags: u8 = if value.drop { 1 } else { 0 }; - self.0.set_byte_to_buffer(flags, buffer)?; - self.0.encode(&value.start, buffer)?; - self.0.encode(&value.length, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let drop = if self.drop { 1 } else { 0 }; + let rest = write_array(&[drop], buffer)?; + Ok(map_encode!(rest, self.start, self.length)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let flags = self.0.decode_u8(buffer)?; - let start: u64 = self.0.decode(buffer)?; - let length: u64 = self.0.decode(buffer)?; - Ok(BitfieldUpdate { - drop: flags == 1, - start, - length, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([flags], rest) = take_array::<1>(buffer)?; + let ((start, length), rest) = map_decode!(rest, [u64, u64]); + Ok(( + BitfieldUpdate { + drop: flags & 1 == 1, + start, + length, + }, + rest, + )) } } /// Oplog Entry #[derive(Debug)] -pub struct Entry { +pub(crate) struct Entry { // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, pub(crate) tree_nodes: Vec, @@ -83,82 +90,86 @@ pub struct Entry { pub(crate) bitfield: Option, } -impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Entry) -> Result { - self.0.add_end(1)?; // flags - if !value.user_data.is_empty() { - self.0.preencode(&value.user_data)?; +impl CompactEncoding for Entry { + fn encoded_size(&self) -> Result { + let mut out = 1; // flags + if !self.user_data.is_empty() { + out += self.user_data.encoded_size()?; } - if !value.tree_nodes.is_empty() { - self.preencode(&value.tree_nodes)?; + if !self.tree_nodes.is_empty() { + out += self.tree_nodes.encoded_size()?; } - if let Some(tree_upgrade) = &value.tree_upgrade { - self.preencode(tree_upgrade)?; + if let Some(tree_upgrade) = &self.tree_upgrade { + out += tree_upgrade.encoded_size()?; } - if let Some(bitfield) = &value.bitfield { - self.preencode(bitfield)?; + if let Some(bitfield) = &self.bitfield { + out += bitfield.encoded_size()?; } - Ok(self.end()) + Ok(out) } - fn encode(&mut self, value: &Entry, buffer: &mut [u8]) -> Result { - let start = self.0.start(); - self.0.add_start(1)?; - let mut flags: u8 = 0; - if !value.user_data.is_empty() { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let (flag_buf, mut rest) = take_array_mut::<1>(buffer)?; + let mut flags = 0u8; + if !self.user_data.is_empty() { flags |= 1; - self.0.encode(&value.user_data, buffer)?; + rest = self.user_data.encode(rest)?; } - if !value.tree_nodes.is_empty() { + if !self.tree_nodes.is_empty() { flags |= 2; - self.encode(&value.tree_nodes, buffer)?; + rest = self.tree_nodes.encode(rest)?; } - if let Some(tree_upgrade) = &value.tree_upgrade { + if let Some(tree_upgrade) = &self.tree_upgrade { flags |= 4; - self.encode(tree_upgrade, buffer)?; + rest = tree_upgrade.encode(rest)?; } - if let Some(bitfield) = &value.bitfield { + if let Some(bitfield) = &self.bitfield { flags |= 8; - self.encode(bitfield, buffer)?; + rest = bitfield.encode(rest)?; } - - buffer[start] = flags; - Ok(self.0.start()) + flag_buf[0] = flags; + Ok(rest) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let flags = self.0.decode_u8(buffer)?; - let user_data: Vec = if flags & 1 != 0 { - self.0.decode(buffer)? + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([flags], rest) = take_array::<1>(buffer)?; + let (user_data, rest) = if flags & 1 != 0 { + >::decode(rest)? } else { - vec![] + (Default::default(), rest) }; - let tree_nodes: Vec = if flags & 2 != 0 { - self.decode(buffer)? + let (tree_nodes, rest) = if flags & 2 != 0 { + >::decode(rest)? } else { - vec![] + (Default::default(), rest) }; - let tree_upgrade: Option = if flags & 4 != 0 { - let value: EntryTreeUpgrade = self.decode(buffer)?; - Some(value) + let (tree_upgrade, rest) = if flags & 2 != 0 { + let (x, rest) = EntryTreeUpgrade::decode(rest)?; + (Some(x), rest) } else { - None + (Default::default(), rest) }; - let bitfield: Option = if flags & 8 != 0 { - let value: BitfieldUpdate = self.decode(buffer)?; - Some(value) + let (bitfield, rest) = if flags & 2 != 0 { + let (x, rest) = BitfieldUpdate::decode(rest)?; + (Some(x), rest) } else { - None + (Default::default(), rest) }; - Ok(Entry { - user_data, - tree_nodes, - tree_upgrade, - bitfield, - }) + Ok(( + Self { + user_data, + tree_nodes, + tree_upgrade, + bitfield, + }, + rest, + )) } } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index aa27dce..c7ad6c2 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,7 +1,8 @@ -use compact_encoding::EncodingErrorKind; -use compact_encoding::{CompactEncoding, EncodingError, State}; +use compact_encoding::{ + decode_usize, map_decode, take_array, write_array, CompactEncoding, EncodingError, +}; +use compact_encoding::{map_encode, sum_encoded_size}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use std::convert::TryInto; use crate::crypto::default_signer_manifest; use crate::crypto::Manifest; @@ -81,86 +82,108 @@ impl HeaderTree { } } -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTree) -> Result { - self.preencode(&value.fork)?; - self.preencode(&value.length)?; - self.preencode(&value.root_hash)?; - self.preencode(&value.signature) +impl CompactEncoding for HeaderTree { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!( + self.fork, + self.length, + self.root_hash, + self.signature + )) } - fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) -> Result { - self.encode(&value.fork, buffer)?; - self.encode(&value.length, buffer)?; - self.encode(&value.root_hash, buffer)?; - self.encode(&value.signature, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!( + buffer, + self.fork, + self.length, + self.root_hash, + self.signature + )) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let fork: u64 = self.decode(buffer)?; - let length: u64 = self.decode(buffer)?; - let root_hash: Box<[u8]> = self.decode(buffer)?; - let signature: Box<[u8]> = self.decode(buffer)?; - Ok(HeaderTree { - fork, - length, - root_hash, - signature, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((fork, length, root_hash, signature), rest) = + map_decode!(buffer, [u64, u64, Box<[u8]>, Box<[u8]>]); + Ok(( + Self { + fork, + length, + root_hash, + signature, + }, + rest, + )) } } /// NB: In Javascript's sodium the secret key contains in itself also the public key, so to /// maintain binary compatibility, we store the public key in the oplog now twice. -impl CompactEncoding for State { - fn preencode(&mut self, value: &PartialKeypair) -> Result { - self.add_end(1 + PUBLIC_KEY_LENGTH)?; - match &value.secret { - Some(_) => { - // Also add room for the public key - self.add_end(1 + SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH) - } - None => self.add_end(1), - } +impl CompactEncoding for PartialKeypair { + fn encoded_size(&self) -> Result { + Ok(1 // len of public key + + PUBLIC_KEY_LENGTH // public key bytes + + match self.secret { + // Secret key contains the public key + Some(_) => 1 + SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH, + None => 1, + }) } - fn encode( - &mut self, - value: &PartialKeypair, - buffer: &mut [u8], - ) -> Result { - let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); - self.encode(&public_key_bytes, buffer)?; - match &value.secret { - Some(secret_key) => { - let mut secret_key_bytes: Vec = - Vec::with_capacity(SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH); - secret_key_bytes.extend_from_slice(&secret_key.to_bytes()); - secret_key_bytes.extend_from_slice(&public_key_bytes); - let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); - self.encode(&secret_key_bytes, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let public_key = self.public.as_bytes().to_vec(); + let rest = public_key.encode(buffer)?; + match &self.secret { + Some(sk) => { + let sk_bytes = [&sk.to_bytes()[..], &public_key[..]].concat(); + sk_bytes.encode(rest) } - None => self.set_byte_to_buffer(0, buffer), + None => write_array(&[0], rest), } } - fn decode(&mut self, buffer: &[u8]) -> Result { - let public_key_bytes: Box<[u8]> = self.decode(buffer)?; - let public_key_bytes: [u8; PUBLIC_KEY_LENGTH] = - public_key_bytes[0..PUBLIC_KEY_LENGTH].try_into().unwrap(); - let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; - let secret: Option = if secret_key_bytes.is_empty() { - None - } else { - let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = - secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); - Some(SigningKey::from_bytes(&secret_key_bytes)) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + // the ful secret/private key contains the public key duplicated in it + const FULL_SIGNING_KEY_LENGTH: usize = SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH; + let (pk_len, rest) = decode_usize(buffer)?; + let (public, rest) = match pk_len { + PUBLIC_KEY_LENGTH => { + let (pk_bytes, rest) = take_array::(rest)?; + let public = VerifyingKey::from_bytes(&pk_bytes).map_err(|e| { + EncodingError::invalid_data(&format!( + "Could not decode public key. error: [{e}]" + )) + })?; + (public, rest) + } + len => { + return Err(EncodingError::invalid_data(&format!( + "Incorrect public key length while decoding. length = [{len}] expected [{PUBLIC_KEY_LENGTH}]" + ))) + } }; - - Ok(PartialKeypair { - public: VerifyingKey::from_bytes(&public_key_bytes).unwrap(), - secret, - }) + let (sk_len, rest) = decode_usize(rest)?; + let (secret, rest) = match sk_len { + 0 => (None, rest), + // full signing key = secret_key.cocat(public_key) + FULL_SIGNING_KEY_LENGTH => { + let (full_key_bytes, rest) = take_array::(rest)?; + let (sk_bytes, _pk_bytes) = take_array::(&full_key_bytes)?; + (Some(SigningKey::from_bytes(&sk_bytes)), rest) + } + len => { + return Err(EncodingError::invalid_data(&format!( + "Incorrect secret key length while decoding. length = [{len}] expected [{FULL_SIGNING_KEY_LENGTH}]" + ))) + } + }; + Ok((PartialKeypair { public, secret }, rest)) } } @@ -171,154 +194,162 @@ pub(crate) struct HeaderHints { pub(crate) contiguous_length: u64, } -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderHints) -> Result { - self.preencode(&value.reorgs)?; - self.preencode(&value.contiguous_length) +impl CompactEncoding for HeaderHints { + fn encoded_size(&self) -> Result { + Ok(sum_encoded_size!(self.reorgs, self.contiguous_length)) } - fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) -> Result { - self.encode(&value.reorgs, buffer)?; - self.encode(&value.contiguous_length, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + Ok(map_encode!(buffer, self.reorgs, self.contiguous_length)) } - fn decode(&mut self, buffer: &[u8]) -> Result { - Ok(HeaderHints { - reorgs: self.decode(buffer)?, - contiguous_length: self.decode(buffer)?, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ((reorgs, contiguous_length), rest) = map_decode!(buffer, [Vec, u64]); + Ok(( + Self { + reorgs, + contiguous_length, + }, + rest, + )) } } -impl CompactEncoding
for State { - fn preencode(&mut self, value: &Header) -> Result { - self.add_end(1)?; // Version - self.add_end(1)?; // Flags - self.preencode_fixed_32()?; // key - self.preencode(&value.manifest)?; - self.preencode(&value.key_pair)?; - self.preencode(&value.user_data)?; - self.preencode(&value.tree)?; - self.preencode(&value.hints) +impl CompactEncoding for Header { + fn encoded_size(&self) -> Result { + Ok(1 + 1 + + 32 + + sum_encoded_size!( + self.manifest, + self.key_pair, + self.user_data, + self.tree, + self.hints + )) } - fn encode(&mut self, value: &Header, buffer: &mut [u8]) -> Result { - self.set_byte_to_buffer(1, buffer)?; // Version - let flags: u8 = 2 | 4; // Manifest and key pair, TODO: external=1 - self.set_byte_to_buffer(flags, buffer)?; - self.encode_fixed_32(&value.key, buffer)?; - self.encode(&value.manifest, buffer)?; - self.encode(&value.key_pair, buffer)?; - self.encode(&value.user_data, buffer)?; - self.encode(&value.tree, buffer)?; - self.encode(&value.hints, buffer) + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = write_array(&[1, 2 | 4], buffer)?; + Ok(map_encode!( + rest, + self.key, + self.manifest, + self.key_pair, + self.user_data, + self.tree, + self.hints + )) } - fn decode(&mut self, buffer: &[u8]) -> Result { - let version: u8 = self.decode_u8(buffer)?; - if version != 1 { - panic!("Unknown oplog version {}", version); - } - let _flags: u8 = self.decode_u8(buffer)?; - let key: [u8; 32] = self - .decode_fixed_32(buffer)? - .to_vec() - .try_into() - .map_err(|_err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - "Invalid key in oplog header", - ) - })?; - let manifest: Manifest = self.decode(buffer)?; - let key_pair: PartialKeypair = self.decode(buffer)?; - let user_data: Vec = self.decode(buffer)?; - let tree: HeaderTree = self.decode(buffer)?; - let hints: HeaderHints = self.decode(buffer)?; - - Ok(Header { - key, - manifest, - key_pair, - user_data, - tree, - hints, - }) + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([_version, _flags], rest) = take_array::<2>(buffer)?; + let (key, rest) = take_array::<32>(rest)?; + let ((manifest, key_pair, user_data, tree, hints), rest) = map_decode!( + rest, [ + Manifest, PartialKeypair, Vec, HeaderTree, HeaderHints + ] + ); + Ok(( + Header { + key, + manifest, + key_pair, + user_data, + tree, + hints, + }, + rest, + )) } } #[cfg(test)] mod tests { + use compact_encoding::{map_decode, to_encoded_bytes}; + use super::*; use crate::crypto::generate_signing_key; #[test] fn encode_partial_key_pair() -> Result<(), EncodingError> { - let mut enc_state = State::new(); let signing_key = generate_signing_key(); let key_pair = PartialKeypair { public: signing_key.verifying_key(), secret: Some(signing_key), }; - enc_state.preencode(&key_pair)?; - let mut buffer = enc_state.create_buffer(); - // Pub key: 1 byte for length, 32 bytes for content - // Sec key: 1 byte for length, 64 bytes for data + + // sizeof(pk.len()) + sizeof(pk) + sizeof(sk.len() + sizeof(sk) let expected_len = 1 + 32 + 1 + 64; - assert_eq!(buffer.len(), expected_len); - assert_eq!(enc_state.end(), expected_len); - assert_eq!(enc_state.start(), 0); - enc_state.encode(&key_pair, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let key_pair_ret: PartialKeypair = dec_state.decode(&buffer)?; - assert_eq!(key_pair.public, key_pair_ret.public); + let encoded = to_encoded_bytes!(&key_pair); + assert_eq!(encoded.len(), expected_len); + let ((dec_kp,), rest) = map_decode!(&encoded, [PartialKeypair]); + assert!(rest.is_empty()); + assert_eq!(key_pair.public, dec_kp.public); assert_eq!( key_pair.secret.unwrap().to_bytes(), - key_pair_ret.secret.unwrap().to_bytes() + dec_kp.secret.unwrap().to_bytes() ); Ok(()) } #[test] fn encode_tree() -> Result<(), EncodingError> { - let mut enc_state = State::new(); let tree = HeaderTree::new(); - enc_state.preencode(&tree)?; - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&tree, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let tree_ret: HeaderTree = dec_state.decode(&buffer)?; - assert_eq!(tree, tree_ret); + let encoded = to_encoded_bytes!(tree); + // all sizeof(0) + sizeof(0) + sizeof(vec![]) + sizeof(vec![]) == 4 + assert_eq!(encoded.len(), 4); + let ((dec_tree,), rest) = map_decode!(&encoded, [HeaderTree]); + assert!(rest.is_empty()); + assert_eq!(dec_tree, tree); + Ok(()) + } + + #[test] + fn encode_tree_with_data() -> Result<(), EncodingError> { + let tree = HeaderTree { + fork: 520, + length: 647, + root_hash: vec![12; 464].into_boxed_slice(), + signature: vec![46; 22].into_boxed_slice(), + }; + let encoded = to_encoded_bytes!(&tree); + let ((dec_tree,), rest) = map_decode!(&encoded, [HeaderTree]); + assert!(rest.is_empty()); + assert_eq!(dec_tree, tree); Ok(()) } #[test] fn encode_header() -> Result<(), EncodingError> { - let mut enc_state = State::new(); + //let mut enc_state = State::new(); let signing_key = generate_signing_key(); let signing_key = PartialKeypair { public: signing_key.verifying_key(), secret: Some(signing_key), }; let header = Header::new(signing_key); - enc_state.preencode(&header)?; - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let header_ret: Header = dec_state.decode(&buffer)?; - assert_eq!(header.key_pair.public, header_ret.key_pair.public); - assert_eq!(header.tree.fork, header_ret.tree.fork); - assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.manifest.hash, header_ret.manifest.hash); + let encoded = to_encoded_bytes!(&header); + let ((dec_header,), rest) = map_decode!(&encoded, [Header]); + assert!(rest.is_empty()); + assert_eq!(header.key_pair.public, dec_header.key_pair.public); + assert_eq!(header.tree.fork, dec_header.tree.fork); + assert_eq!(header.tree.length, dec_header.tree.length); + assert_eq!(header.tree.length, dec_header.tree.length); + assert_eq!(header.manifest.hash, dec_header.manifest.hash); assert_eq!( header.manifest.signer.public_key, - header_ret.manifest.signer.public_key + dec_header.manifest.signer.public_key ); assert_eq!( header.manifest.signer.signature, - header_ret.manifest.signer.signature + dec_header.manifest.signer.signature ); Ok(()) } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 6c72020..eb64cfb 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,12 +1,15 @@ +use compact_encoding::{ + as_array_mut, get_slices_checked, get_slices_mut_checked, map_decode, take_array_mut, + CompactEncoding, FixedWidthEncoding, FixedWidthU32, +}; use futures::future::Either; use std::convert::{TryFrom, TryInto}; use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; -use crate::encoding::{CompactEncoding, HypercoreState}; use crate::tree::MerkleTreeChangeset; use crate::{HypercoreError, Node, PartialKeypair}; -mod entry; +pub(crate) mod entry; mod header; pub(crate) use entry::{Entry, EntryTreeUpgrade}; @@ -15,6 +18,13 @@ pub(crate) use header::{Header, HeaderTree}; pub(crate) const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; const HEADER_SIZE: usize = 4096; +// NB: we use the word "leader" to describe the 8 byte part put before a chunk of data that +// contains 4 byte checksum, then a 30 bit unsigned integer, then a "header bit" and a "partial +// bit" +const CRC_SIZE: usize = 4; +const LEN_PARTIAL_AND_HEADER_INFO_SIZE: usize = 4; +const LEADER_SIZE: usize = CRC_SIZE + LEN_PARTIAL_AND_HEADER_INFO_SIZE; + /// Oplog. /// /// There are two memory areas for an `Header` in `RandomAccessStorage`: one is the current @@ -73,8 +83,8 @@ enum OplogSlot { } #[derive(Debug)] -struct ValidateLeaderOutcome { - state: HypercoreState, +struct ValidateLeaderOutcome<'a> { + state: &'a [u8], header_bit: bool, partial_bit: bool, } @@ -96,25 +106,35 @@ impl Oplog { Some(info) => { let existing = info.data.expect("Could not get data of existing oplog"); // First read and validate both headers stored in the existing oplog - let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; - let h2_outcome = - Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; - + let h1_outcome = if let Some(h1) = + existing.get(OplogSlot::FirstHeader as usize..OplogSlot::SecondHeader as usize) + { + Self::validate_leader(h1)? + } else { + None + }; + let h2_outcome = if let Some(h2) = + existing.get(OplogSlot::SecondHeader as usize..OplogSlot::Entries as usize) + { + Self::validate_leader(h2)? + } else { + None + }; // Depending on what is stored, the state needs to be set accordingly. // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. - let mut outcome: OplogOpenOutcome = if let Some(mut h1_outcome) = h1_outcome { + let mut outcome: OplogOpenOutcome = if let Some(h1_outcome) = h1_outcome { let (header, header_bits): (Header, [bool; 2]) = - if let Some(mut h2_outcome) = h2_outcome { + if let Some(h2_outcome) = h2_outcome { let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; let header: Header = if header_bits[0] == header_bits[1] { - (*h1_outcome.state).decode(&existing)? + Header::decode(h1_outcome.state)?.0 } else { - (*h2_outcome.state).decode(&existing)? + Header::decode(h2_outcome.state)?.0 }; (header, header_bits) } else { ( - (*h1_outcome.state).decode(&existing)?, + Header::decode(h1_outcome.state)?.0, [h1_outcome.header_bit, h1_outcome.header_bit], ) }; @@ -124,7 +144,7 @@ impl Oplog { entries_byte_length: 0, }; OplogOpenOutcome::new(oplog, header, Box::new([])) - } else if let Some(mut h2_outcome) = h2_outcome { + } else if let Some(h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot // but Javascript supports this so we should too. let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; @@ -133,11 +153,7 @@ impl Oplog { entries_length: 0, entries_byte_length: 0, }; - OplogOpenOutcome::new( - oplog, - (*h2_outcome.state).decode(&existing)?, - Box::new([]), - ) + OplogOpenOutcome::new(oplog, Header::decode(h2_outcome.state)?.0, Box::new([])) } else if let Some(key_pair) = key_pair { // There is nothing in the oplog, start from fresh given key pair. Self::fresh(key_pair.clone())? @@ -150,16 +166,15 @@ impl Oplog { // Read headers that might be stored in the existing content if existing.len() > OplogSlot::Entries as usize { - let mut entry_offset = OplogSlot::Entries as usize; + let mut entries_buff = + get_slices_checked(&existing, OplogSlot::Entries as usize)?.1; let mut entries: Vec = Vec::new(); let mut partials: Vec = Vec::new(); - while let Some(mut entry_outcome) = - Self::validate_leader(entry_offset, &existing)? - { - let entry: Entry = entry_outcome.state.decode(&existing)?; - entries.push(entry); + while let Some(entry_outcome) = Self::validate_leader(entries_buff)? { + let res = Entry::decode(entry_outcome.state)?; + entries.push(res.0); + entries_buff = res.1; partials.push(entry_outcome.partial_bit); - entry_offset = (*entry_outcome.state).end(); } // Remove all trailing partial entries @@ -264,7 +279,7 @@ impl Oplog { let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits, clear_traces)?; let mut combined_infos_to_flush: Vec = - infos_to_flush.into_vec().drain(0..1).into_iter().collect(); + infos_to_flush.into_vec().drain(0..1).collect(); let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, new_header_bits, clear_traces)?; combined_infos_to_flush.extend(infos_to_flush.into_vec()); @@ -286,31 +301,22 @@ impl Oplog { ) -> Result, HypercoreError> { let len = batch.len(); let header_bit = self.get_current_header_bit(); - // Leave room for leaders - let mut state = HypercoreState::new_with_start_and_end(0, len * 8); - for entry in batch.iter() { - state.preencode(entry)?; + let mut size = len * LEADER_SIZE; + + for e in batch.iter() { + size += e.encoded_size()?; } - let mut buffer = state.create_buffer(); + let mut buffer = vec![0; size]; + let mut rest = buffer.as_mut_slice(); for (i, entry) in batch.iter().enumerate() { - (*state).add_start(8)?; - let start = state.start(); let partial_bit: bool = atomic && i < len - 1; - state.encode(entry, &mut buffer)?; - Self::prepend_leader( - state.start() - start, - header_bit, - partial_bit, - &mut state, - &mut buffer, - )?; + rest = encode_with_leader(entry, partial_bit, header_bit, rest)?; } - let index = OplogSlot::Entries as u64 + self.entries_byte_length; self.entries_length += len as u64; - self.entries_byte_length += buffer.len() as u64; + self.entries_byte_length += size as u64; Ok(vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice()) } @@ -342,8 +348,6 @@ impl Oplog { clear_traces: bool, ) -> Result<([bool; 2], Box<[StoreInfo]>), HypercoreError> { // The first 8 bytes will be filled with `prepend_leader`. - let data_start_index: usize = 8; - let mut state = HypercoreState::new_with_start_and_end(data_start_index, data_start_index); // Get the right slot and header bit let (oplog_slot, header_bit) = @@ -357,32 +361,17 @@ impl Oplog { } } - // Preencode the new header - (*state).preencode(header)?; + let mut size = LEADER_SIZE + header.encoded_size()?; + size += header.encoded_size()?; // If clearing, lets add zeros to the end - let end = if clear_traces { - let end = state.end(); - state.set_end(HEADER_SIZE); - end - } else { - state.end() - }; + if clear_traces { + size = HEADER_SIZE; + } // Create a buffer for the needed data - let mut buffer = state.create_buffer(); - - // Encode the header - (*state).encode(header, &mut buffer)?; - - // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits - Self::prepend_leader( - end - data_start_index, - header_bit, - false, - &mut state, - &mut buffer, - )?; + let mut buffer = vec![0; size]; + encode_with_leader(header, false, header_bit, &mut buffer)?; // The oplog is always truncated to the minimum byte size, which is right after // all of the entries in the oplog finish. @@ -397,76 +386,39 @@ impl Oplog { )) } - /// Prepends given `State` with 4 bytes of CRC followed by 4 bytes containing length of - /// following buffer, 1 bit indicating which header is relevant to the entry (or if used to - /// wrap the actual header, then the header bit relevant for saving) and 1 bit that tells if - /// the written batch is only partially finished. For this to work, the state given must have - /// 8 bytes in reserve in the beginning, so that state.start can be set back 8 bytes. - fn prepend_leader( - len: usize, - header_bit: bool, - partial_bit: bool, - state: &mut HypercoreState, - buffer: &mut Box<[u8]>, - ) -> Result<(), HypercoreError> { - // The 4 bytes right before start of data is the length in 8+8+8+6=30 bits. The 31st bit is - // the partial bit and 32nd bit the header bit. - let start = (*state).start(); - (*state).set_start(start - len - 4)?; - let len_u32: u32 = len.try_into().unwrap(); - let partial_bit: u32 = if partial_bit { 2 } else { 0 }; - let header_bit: u32 = if header_bit { 1 } else { 0 }; - let combined: u32 = (len_u32 << 2) | header_bit | partial_bit; - state.encode_u32(combined, buffer)?; - - // Before that, is a 4 byte CRC32 that is a checksum of the above encoded 4 bytes and the - // content. - let start = state.start(); - state.set_start(start - 8)?; - let checksum = crc32fast::hash(&buffer[state.start() + 4..state.start() + 8 + len]); - state.encode_u32(checksum, buffer)?; - Ok(()) - } - /// Validates that leader at given index is valid, and returns header and partial bits and /// `State` for the header/entry that the leader was for. - fn validate_leader( - index: usize, - buffer: &[u8], - ) -> Result, HypercoreError> { - if buffer.len() < index + 8 { + fn validate_leader(buffer: &[u8]) -> Result>, HypercoreError> { + if buffer.len() < 8 { return Ok(None); } - let mut state = HypercoreState::new_with_start_and_end(index, buffer.len()); - let stored_checksum: u32 = state.decode_u32(buffer)?; - let combined: u32 = state.decode_u32(buffer)?; + let ((stored_checksum, combined), data_buff) = + map_decode!(buffer, [FixedWidthU32<'_>, FixedWidthU32<'_>]); + let len = usize::try_from(combined >> 2) .expect("Attempted converting to a 32 bit usize on below 32 bit system"); // NB: In the Javascript version IIUC zero length is caught only with a mismatch // of checksums, which is silently interpreted to only mean "no value". That doesn't sound good: // better to throw an error on mismatch and let the caller at least log the problem. - if len == 0 || state.end() - state.start() < len { + if len == 0 || data_buff.len() < len { return Ok(None); } + let header_bit = combined & 1 == 1; let partial_bit = combined & 2 == 2; - let new_start = index + 8; - state.set_end(new_start + len); - state.set_start(new_start)?; - - let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end()]); + let to_hash = &buffer[CRC_SIZE..LEADER_SIZE + len]; + let calculated_checksum = crc32fast::hash(to_hash); if calculated_checksum != stored_checksum { return Err(HypercoreError::InvalidChecksum { - context: "Calculated signature does not match oplog signature".to_string(), + context: format!("Calculated signature [{calculated_checksum}] does not match oplog signature [{stored_checksum}]"), }); }; - Ok(Some(ValidateLeaderOutcome { header_bit, partial_bit, - state, + state: data_buff, })) } @@ -493,3 +445,57 @@ impl Oplog { } } } + +/// Create a header. 30 bits are the length plus two bits for "partial" and "header" info +fn build_len_and_info_header(data_length: usize, header_bit: bool, partial_bit: bool) -> u32 { + let data_length: u32 = data_length + .try_into() + .expect("Must be able to convert usize to u32"); + const MASK: u32 = (3u32).rotate_right(2); + + if (MASK & data_length) != 0 { + panic!("Data length would overflow. It does not fit in 30 bits"); + } + let partial_bit: u32 = if partial_bit { 2 } else { 0 }; + let header_bit: u32 = if header_bit { 1 } else { 0 }; + (data_length << 2) | header_bit | partial_bit +} + +fn write_leader_parts( + header_bit: bool, + partial_bit: bool, + crc_zone: &mut [u8; CRC_SIZE], + len_and_meta_zone: &mut [u8; LEN_PARTIAL_AND_HEADER_INFO_SIZE], + data: &[u8], +) -> Result<(), HypercoreError> { + // first we write the length and partial data + let len_and_info = build_len_and_info_header(data.len(), header_bit, partial_bit); + (len_and_info.as_fixed_width()).encode(len_and_meta_zone)?; + // next we hash the new header info along with the data + let mut hasher = crc32fast::Hasher::new(); + hasher.update(len_and_meta_zone); + hasher.update(data); + hasher.finalize().as_fixed_width().encode(crc_zone)?; + Ok(()) +} + +fn encode_with_leader<'a>( + thing: &impl CompactEncoding, + partial_bit: bool, + header_bit: bool, + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], HypercoreError> { + let (leader_bytes, data_and_rest) = take_array_mut::(buffer)?; + let enc_size = thing.encoded_size()?; + let (data_buff, rest) = get_slices_mut_checked(data_and_rest, enc_size)?; + let (crc_zone, len_and_meta_zone) = get_slices_mut_checked(leader_bytes, CRC_SIZE)?; + thing.encode(data_buff)?; + write_leader_parts( + header_bit, + partial_bit, + as_array_mut::(crc_zone)?, + as_array_mut::(len_and_meta_zone)?, + data_buff, + )?; + Ok(rest) +} diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index c957919..87cb22b 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,4 +1,6 @@ -use compact_encoding::State; +use compact_encoding::{ + as_array, map_decode, to_encoded_bytes, EncodingError, FixedWidthEncoding, FixedWidthU64, +}; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; @@ -92,7 +94,7 @@ impl MerkleTree { if length > 0 { length /= 2; } - let signature: Option = if header_tree.signature.len() > 0 { + let signature: Option = if !header_tree.signature.is_empty() { Some( Signature::try_from(&*header_tree.signature).map_err(|_err| { HypercoreError::InvalidSignature { @@ -479,11 +481,7 @@ impl MerkleTree { start: upgrade.start, length: upgrade.length, nodes: p.upgrade.expect("nodes need to be set"), - additional_nodes: if let Some(additional_upgrade) = p.additional_upgrade { - additional_upgrade - } else { - vec![] - }, + additional_nodes: p.additional_upgrade.unwrap_or_default(), signature: signature .expect("signature needs to be set") .to_bytes() @@ -658,13 +656,14 @@ impl MerkleTree { pub(crate) fn flush_nodes(&mut self) -> Vec { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); for (_, node) in self.unflushed.drain() { - let (mut state, mut buffer) = State::new_with_size(40); - state - .encode_u64(node.length, &mut buffer) - .expect("Encoding u64 should not fail"); - state - .encode_fixed_32(&node.hash, &mut buffer) - .expect("Encoding fixed 32 bytes should not fail"); + let buffer = (|| { + let hash = as_array::<32>(&node.hash)?; + Ok::, EncodingError>(to_encoded_bytes!( + node.length.as_fixed_width(), + hash + )) + })() + .expect("Encoding u64 should not fail"); infos_to_flush.push(StoreInfo::new_content( Store::Tree, node.index * 40, @@ -1475,8 +1474,7 @@ fn index_from_info(info: &StoreInfo) -> u64 { fn node_from_bytes(index: &u64, data: &[u8]) -> Result { let len_buf = &data[..8]; let hash = &data[8..]; - let mut state = State::from_buffer(len_buf); - let len = state.decode_u64(len_buf)?; + let len = map_decode!(len_buf, [FixedWidthU64<'_>]).0 .0; Ok(Node::new(*index, hash.to_vec(), len)) } @@ -1564,7 +1562,7 @@ fn parent_node(index: u64, left: &Node, right: &Node) -> Node { ) } -fn block_node(index: u64, value: &Vec) -> Node { +fn block_node(index: u64, value: &[u8]) -> Node { Node::new( index, Hash::data(value).as_bytes().to_vec(),