diff --git a/src/common/io/src/bitmap.rs b/src/common/io/src/bitmap.rs index a837d7eb771e5..fbb3f456ffe2e 100644 --- a/src/common/io/src/bitmap.rs +++ b/src/common/io/src/bitmap.rs @@ -20,6 +20,7 @@ use std::ops::BitAndAssign; use std::ops::BitOrAssign; use std::ops::BitXorAssign; use std::ops::SubAssign; +use std::ptr; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -37,6 +38,10 @@ const HYBRID_HEADER_LEN: usize = 4; type SmallBitmap = SmallVec<[u64; LARGE_THRESHOLD]>; +/// Perf Tips: +/// - The deserialization performance of HybridBitmap significantly impacts the performance of Bitmap-related calculations. +/// - Calculations may frequently create new Bitmaps; reusing them as much as possible can effectively improve performance. +/// - do not use Box to construct HybridBitmap #[allow(clippy::large_enum_variant)] #[derive(Clone)] pub enum HybridBitmap { @@ -241,8 +246,7 @@ impl std::ops::BitOrAssign for HybridBitmap { } } HybridBitmap::Small(lhs_set) => { - let left = mem::take(lhs_set); - *lhs_set = small_union(left, rhs_set.as_slice()); + small_union(lhs_set, rhs_set.as_slice()); if self.len() >= LARGE_THRESHOLD as u64 { let _ = self.promote_to_tree(); } @@ -303,14 +307,14 @@ impl std::ops::BitXorAssign for HybridBitmap { self.try_demote(); } HybridBitmap::Small(rhs_set) => match self { + // Disjoint data in the bitmap can cause lhs_tree expansion during XOR, making this path a significant performance bottleneck. HybridBitmap::Large(lhs_tree) => { let rhs_tree = RoaringTreemap::from_iter(rhs_set.iter().copied()); lhs_tree.bitxor_assign(rhs_tree); self.try_demote(); } HybridBitmap::Small(lhs_set) => { - let result = small_symmetric_difference(lhs_set.as_slice(), rhs_set.as_slice()); - *lhs_set = result; + small_symmetric_difference(lhs_set, rhs_set.as_slice()); if self.len() >= LARGE_THRESHOLD as u64 { let _ = self.promote_to_tree(); } @@ -541,12 +545,11 @@ fn decode_small_bitmap(payload: &[u8]) -> Result { ))); } - let mut data = [0u8; std::mem::size_of::()]; let set: SmallBitmap = bytes - .chunks_exact(data.len()) - .map(move |chunk| { - data.copy_from_slice(chunk); - u64::from_le_bytes(data) + .chunks_exact(std::mem::size_of::()) + .map(|chunk| { + let raw = unsafe { ptr::read_unaligned(chunk.as_ptr() as *const u64) }; + u64::from_le(raw) }) .collect(); Ok(HybridBitmap::Small(set)) @@ -562,37 +565,62 @@ fn small_insert(set: &mut SmallBitmap, value: u64) -> bool { } } -fn small_union(left: SmallBitmap, right: &[u64]) -> SmallBitmap { - if right.is_empty() { - return left; +fn small_union(target: &mut SmallBitmap, other: &[u64]) { + if other.is_empty() { + return; } - if left.is_empty() { - return SmallBitmap::from_slice(right); + if target.is_empty() { + target.extend_from_slice(other); + return; } - let left_slice = left.as_slice(); - let mut result = SmallBitmap::with_capacity(left_slice.len() + right.len()); - let mut i = 0; - let mut j = 0; + let lhs_len = target.len(); + let rhs_len = other.len(); + target.reserve(rhs_len); + let mut write = lhs_len + rhs_len; + target.resize(write, 0); - while i < left_slice.len() && j < right.len() { - let lv = left_slice[i]; - let rv = right[j]; - if lv < rv { - result.push(lv); - i += 1; - } else if rv < lv { - result.push(rv); - j += 1; - } else { - result.push(lv); - i += 1; - j += 1; + let mut i = lhs_len; + let mut j = rhs_len; + + while i > 0 && j > 0 { + let lv = target[i - 1]; + let rv = other[j - 1]; + write -= 1; + match lv.cmp(&rv) { + std::cmp::Ordering::Greater => { + target[write] = lv; + i -= 1; + } + std::cmp::Ordering::Less => { + target[write] = rv; + j -= 1; + } + std::cmp::Ordering::Equal => { + target[write] = lv; + i -= 1; + j -= 1; + } } } - result.extend_from_slice(&left_slice[i..]); - result.extend_from_slice(&right[j..]); - result + + while i > 0 { + write -= 1; + target[write] = target[i - 1]; + i -= 1; + } + + while j > 0 { + write -= 1; + target[write] = other[j - 1]; + j -= 1; + } + + if write > 0 { + let len = target.len(); + target.copy_within(write..len, 0); + target.truncate(len - write); + } } fn small_intersection(lhs: &mut SmallBitmap, rhs: &mut SmallBitmap) { @@ -673,28 +701,62 @@ fn small_difference(lhs: &[u64], rhs: &[u64]) -> SmallBitmap { result } -fn small_symmetric_difference(lhs: &[u64], rhs: &[u64]) -> SmallBitmap { - let mut result = SmallBitmap::with_capacity(lhs.len() + rhs.len()); - let mut i = 0; - let mut j = 0; +fn small_symmetric_difference(target: &mut SmallBitmap, other: &[u64]) { + if other.is_empty() { + return; + } + if target.is_empty() { + target.extend_from_slice(other); + return; + } - while i < lhs.len() && j < rhs.len() { - let lv = lhs[i]; - let rv = rhs[j]; - if lv < rv { - result.push(lv); - i += 1; - } else if rv < lv { - result.push(rv); - j += 1; - } else { - i += 1; - j += 1; + let lhs_len = target.len(); + let rhs_len = other.len(); + target.reserve(rhs_len); + let mut write = lhs_len + rhs_len; + target.resize(write, 0); + + let mut i = lhs_len; + let mut j = rhs_len; + + while i > 0 && j > 0 { + let lv = target[i - 1]; + let rv = other[j - 1]; + match lv.cmp(&rv) { + std::cmp::Ordering::Greater => { + write -= 1; + target[write] = lv; + i -= 1; + } + std::cmp::Ordering::Less => { + write -= 1; + target[write] = rv; + j -= 1; + } + std::cmp::Ordering::Equal => { + i -= 1; + j -= 1; + } } } - result.extend_from_slice(&lhs[i..]); - result.extend_from_slice(&rhs[j..]); - result + + while i > 0 { + write -= 1; + target[write] = target[i - 1]; + i -= 1; + } + + while j > 0 { + write -= 1; + target[write] = other[j - 1]; + j -= 1; + } + + if write > 0 { + let len = target.len(); + target.copy_within(write..len, 0); + target.truncate(len - write); + } } fn small_is_superset(lhs: &SmallBitmap, rhs: &SmallBitmap) -> bool { @@ -759,10 +821,10 @@ mod tests { #[test] fn small_union_merges_and_deduplicates() { - let left: SmallBitmap = smallvec![1_u64, 3, 5]; + let mut left: SmallBitmap = smallvec![1_u64, 3, 5]; let right = [0_u64, 3, 4, 7]; - let result = small_union(left, &right); - assert_eq!(result.as_slice(), &[0, 1, 3, 4, 5, 7]); + small_union(&mut left, &right); + assert_eq!(left.as_slice(), &[0, 1, 3, 4, 5, 7]); } #[test] @@ -798,10 +860,10 @@ mod tests { #[test] fn small_symmetric_difference_handles_overlap() { - let lhs = [1_u64, 2, 4]; + let mut lhs: SmallBitmap = smallvec![1_u64, 2, 4]; let rhs = [2_u64, 3, 5]; - let result = small_symmetric_difference(&lhs, &rhs); - assert_eq!(result.as_slice(), &[1, 3, 4, 5]); + small_symmetric_difference(&mut lhs, &rhs); + assert_eq!(lhs.as_slice(), &[1, 3, 4, 5]); } #[test] diff --git a/src/query/functions/benches/bench.rs b/src/query/functions/benches/bench.rs index 96e7eef3d9438..198105930095d 100644 --- a/src/query/functions/benches/bench.rs +++ b/src/query/functions/benches/bench.rs @@ -30,18 +30,11 @@ fn main() { #[divan::bench_group(max_time = 0.5)] mod dummy { use databend_common_expression::type_check; - use databend_common_expression::types::BitmapType; - use databend_common_expression::BlockEntry; - use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::Evaluator; - use databend_common_expression::FromData; use databend_common_expression::FunctionContext; - use databend_common_functions::aggregates::eval_aggr_for_test; use databend_common_functions::test_utils as parser; use databend_common_functions::BUILTIN_FUNCTIONS; - use databend_common_io::deserialize_bitmap; - use databend_common_io::HybridBitmap; #[divan::bench(args = [10240, 102400])] fn parse(bencher: divan::Bencher, n: usize) { @@ -74,6 +67,59 @@ mod dummy { let _ = divan::black_box(evaluator.run(&expr)); }); } +} + +#[divan::bench_group(max_time = 0.5)] +mod bitmap { + use databend_common_expression::types::number::UInt64Type; + use databend_common_expression::types::BitmapType; + use databend_common_expression::BlockEntry; + use databend_common_expression::Column; + use databend_common_expression::FromData; + use databend_common_functions::aggregates::eval_aggr_for_test; + use databend_common_io::deserialize_bitmap; + use databend_common_io::HybridBitmap; + + fn expected_xor_values(rows: usize) -> Vec { + const PERIOD: usize = 15; + + fn parity_for_rows(count: usize) -> [u8; 5] { + let mut parity = [0u8; 5]; + for n in 0..count { + let mut inserted = [false; 5]; + inserted[1] = true; + inserted[n % 3] = true; + let v5 = n % 5; + inserted[v5] = true; + for (idx, flag) in inserted.iter().enumerate() { + if *flag { + parity[idx] ^= 1; + } + } + } + parity + } + + let block_parity = parity_for_rows(PERIOD); + let mut parity = [0u8; 5]; + let full_blocks = rows / PERIOD; + if full_blocks % 2 == 1 { + for (dst, src) in parity.iter_mut().zip(block_parity.iter()) { + *dst ^= *src; + } + } + let remainder = rows % PERIOD; + let rem_parity = parity_for_rows(remainder); + for (dst, src) in parity.iter_mut().zip(rem_parity.iter()) { + *dst ^= *src; + } + + parity + .iter() + .enumerate() + .filter_map(|(value, bit)| (*bit == 1).then_some(value as u64)) + .collect() + } fn build_bitmap_column(rows: u64) -> Column { let bitmaps = (0..rows) @@ -92,35 +138,194 @@ mod dummy { BitmapType::from_data(bitmaps) } - #[divan::bench(args = [100_000, 10_000_000])] + fn build_disjoint_bitmap_column(rows: u64) -> Column { + let bitmaps = (0..rows) + .map(|number| { + let mut rb = HybridBitmap::new(); + let base = number * 2; + rb.insert(base); + rb.insert(base + 1); + + let mut data = Vec::new(); + rb.serialize_into(&mut data).unwrap(); + data + }) + .collect(); + + BitmapType::from_data(bitmaps) + } + + fn build_uint64_column(rows: usize, generator: F) -> Column + where F: FnMut(u64) -> u64 { + let data: Vec = (0..rows as u64).map(generator).collect(); + UInt64Type::from_data(data) + } + + fn eval_bitmap_result(entry: &BlockEntry, rows: usize, agg_name: &'static str) -> HybridBitmap { + let (result_column, _) = eval_aggr_for_test( + agg_name, + vec![], + std::slice::from_ref(entry), + rows, + false, + vec![], + ) + .unwrap_or_else(|_| panic!("{agg_name} evaluation failed")); + + let Column::Bitmap(result) = result_column.remove_nullable() else { + panic!("{agg_name} should return a Bitmap column"); + }; + let Some(bytes) = result.index(0) else { + panic!("{agg_name} should return exactly one row"); + }; + deserialize_bitmap(bytes).expect("deserialize bitmap result") + } + + fn run_bitmap_result_bench( + bencher: divan::Bencher, + rows: usize, + agg_name: &'static str, + entry: &BlockEntry, + validator: F, + ) where + F: Fn(&HybridBitmap) + Sync, + { + bencher.bench(|| { + let rb = eval_bitmap_result(entry, rows, agg_name); + validator(&rb); + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] fn bitmap_intersect(bencher: divan::Bencher, rows: usize) { // Emulate `CREATE TABLE ... AS SELECT build_bitmap` // followed by `SELECT bitmap_intersect(a) FROM c`. let column = build_bitmap_column(rows as u64); let entry: BlockEntry = column.into(); - bencher.bench(|| { - let (result_column, _) = eval_aggr_for_test( - "bitmap_intersect", - vec![], - std::slice::from_ref(&entry), - rows, - false, - vec![], - ) - .expect("bitmap_intersect evaluation"); - - let Column::Bitmap(result) = result_column.remove_nullable() else { - panic!("bitmap_intersect should return a Bitmap column"); - }; - let Some(bytes) = result.index(0) else { - panic!("result should contain exactly one row"); - }; - let rb = deserialize_bitmap(bytes).expect("deserialize bitmap result"); + run_bitmap_result_bench(bencher, rows, "bitmap_intersect", &entry, |rb| { assert_eq!(rb.len(), 1); assert!(rb.contains(1)); }); } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_union(bencher: divan::Bencher, rows: usize) { + let column = build_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_union", &entry, |rb| { + assert_eq!(rb.len(), 5); + for value in 0..5 { + assert!(rb.contains(value), "bitmap_union missing {value}"); + } + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_or_agg(bencher: divan::Bencher, rows: usize) { + let column = build_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_or_agg", &entry, |rb| { + assert_eq!(rb.len(), 5); + for value in 0..5 { + assert!(rb.contains(value), "bitmap_or_agg missing {value}"); + } + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_and_agg(bencher: divan::Bencher, rows: usize) { + let column = build_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_and_agg", &entry, |rb| { + assert_eq!(rb.len(), 1); + assert!(rb.contains(1)); + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_intersect_empty(bencher: divan::Bencher, rows: usize) { + let column = build_disjoint_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_intersect", &entry, |rb| { + assert_eq!(rb.len(), 0, "intersection should be empty"); + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_union_disjoint(bencher: divan::Bencher, rows: usize) { + let column = build_disjoint_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_union", &entry, |rb| { + let expected = rows as u64 * 2; + assert_eq!(rb.len(), expected); + if expected > 0 { + assert!(rb.contains(0)); + assert!(rb.contains(expected - 1)); + } + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_xor_agg(bencher: divan::Bencher, rows: usize) { + let column = build_disjoint_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_xor_agg", &entry, |rb| { + let expected = rows as u64 * 2; + assert_eq!(rb.len(), expected); + if expected > 0 { + assert!(rb.contains(0)); + assert!(rb.contains(expected - 1)); + } + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_xor_agg_overlap(bencher: divan::Bencher, rows: usize) { + let column = build_bitmap_column(rows as u64); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_xor_agg", &entry, |rb| { + let expected = expected_xor_values(rows); + let actual: Vec = rb.iter().collect(); + assert_eq!(actual, expected); + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_construct_agg_dense(bencher: divan::Bencher, rows: usize) { + let column = build_uint64_column(rows, |value| value); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_construct_agg", &entry, |rb| { + let expected = rows as u64; + assert_eq!(rb.len(), expected); + if expected > 0 { + assert!(rb.contains(expected / 2)); + } + }); + } + + #[divan::bench(args = [100_000, 1_000_000])] + fn bitmap_construct_agg_repeating(bencher: divan::Bencher, rows: usize) { + const CARDINALITY: u64 = 1024; + let column = build_uint64_column(rows, |value| value % CARDINALITY); + let entry: BlockEntry = column.into(); + + run_bitmap_result_bench(bencher, rows, "bitmap_construct_agg", &entry, |rb| { + let expected = CARDINALITY.min(rows as u64); + assert_eq!(rb.len(), expected); + if expected > 0 { + assert!(rb.contains(expected - 1)); + } + }); + } } #[divan::bench_group(max_time = 0.5)]