From 9b62a13d5243c4cd38d7b3571e1eb785fbba1650 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Sun, 15 Mar 2026 17:22:07 +0000 Subject: [PATCH 01/12] init --- .../src/account_compression_cpi/nullify.rs | 29 ++++++++++++++++++- .../src/account_compression_cpi/sdk.rs | 16 ++++++---- programs/registry/src/errors.rs | 2 ++ programs/registry/src/lib.rs | 25 ++++++++++++++++ 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index 818e2b43a8..badf4fdf7f 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -3,7 +3,7 @@ use account_compression::{ }; use anchor_lang::prelude::*; -use crate::epoch::register_epoch::ForesterEpochPda; +use crate::{epoch::register_epoch::ForesterEpochPda, errors::RegistryError}; #[derive(Accounts)] pub struct NullifyLeaves<'info> { @@ -61,3 +61,30 @@ pub fn process_nullify( proofs, ) } + +pub fn process_nullify_from_remaining_accounts( + ctx: &Context, + bump: u8, + change_log_indices: Vec, + leaves_queue_indices: Vec, + indices: Vec, +) -> Result<()> { + if ctx.remaining_accounts.is_empty() { + return err!(RegistryError::EmptyProofAccounts); + } + + let proof_nodes: Vec<[u8; 32]> = ctx + .remaining_accounts + .iter() + .map(|account_info| account_info.key().to_bytes()) + .collect(); + + process_nullify( + ctx, + bump, + change_log_indices, + leaves_queue_indices, + indices, + vec![proof_nodes], + ) +} diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index f002c35499..057dcbf7ef 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -9,7 +9,7 @@ use light_batched_merkle_tree::{ initialize_state_tree::InitStateTreeAccountsInstructionData, }; use light_system_program::program::LightSystemProgram; -use solana_sdk::instruction::Instruction; +use solana_sdk::instruction::{AccountMeta, Instruction}; use crate::utils::{ get_cpi_authority_pda, get_forester_epoch_pda_from_authority, get_protocol_config_pda_address, @@ -37,15 +37,14 @@ pub fn create_nullify_instruction( Some(get_forester_epoch_pda_from_authority(&inputs.derivation, epoch).0) }; let (cpi_authority, bump) = get_cpi_authority_pda(); - let instruction_data = crate::instruction::Nullify { + let instruction_data = crate::instruction::NullifyWithProofAccounts { bump, change_log_indices: inputs.change_log_indices, leaves_queue_indices: inputs.leaves_queue_indices, indices: inputs.indices, - proofs: inputs.proofs, }; - let accounts = crate::accounts::NullifyLeaves { + let base_accounts = crate::accounts::NullifyLeaves { authority: inputs.authority, registered_forester_pda, registered_program_pda: register_program_pda, @@ -55,9 +54,16 @@ pub fn create_nullify_instruction( cpi_authority, account_compression_program: account_compression::ID, }; + let mut accounts = base_accounts.to_account_metas(Some(true)); + for proof in inputs.proofs { + for node in proof { + accounts.push(AccountMeta::new_readonly(Pubkey::new_from_array(node), false)); + } + } + Instruction { program_id: crate::ID, - accounts: accounts.to_account_metas(Some(true)), + accounts, data: instruction_data.data(), } } diff --git a/programs/registry/src/errors.rs b/programs/registry/src/errors.rs index 7c445d2ca3..abe4318fb9 100644 --- a/programs/registry/src/errors.rs +++ b/programs/registry/src/errors.rs @@ -38,6 +38,8 @@ pub enum RegistryError { InvalidTokenAccountData, #[msg("Indices array cannot be empty")] EmptyIndices, + #[msg("Proof accounts cannot be empty")] + EmptyProofAccounts, #[msg("Failed to borrow account data")] BorrowAccountDataFailed, #[msg("Failed to serialize instruction data")] diff --git a/programs/registry/src/lib.rs b/programs/registry/src/lib.rs index a21b58cd4b..3caa70e025 100644 --- a/programs/registry/src/lib.rs +++ b/programs/registry/src/lib.rs @@ -420,6 +420,31 @@ pub mod light_registry { ) } + pub fn nullify_with_proof_accounts<'info>( + ctx: Context<'_, '_, '_, 'info, NullifyLeaves<'info>>, + bump: u8, + change_log_indices: Vec, + leaves_queue_indices: Vec, + indices: Vec, + ) -> Result<()> { + let metadata = ctx.accounts.merkle_tree.load()?.metadata; + check_forester( + &metadata, + ctx.accounts.authority.key(), + ctx.accounts.nullifier_queue.key(), + &mut ctx.accounts.registered_forester_pda, + DEFAULT_WORK_V1, + )?; + + process_nullify_from_remaining_accounts( + &ctx, + bump, + change_log_indices, + leaves_queue_indices, + indices, + ) + } + #[allow(clippy::too_many_arguments)] pub fn update_address_merkle_tree( ctx: Context, From 67d65344dae616f9c868ff98273d2d701ec97c98 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Sun, 15 Mar 2026 18:29:20 +0000 Subject: [PATCH 02/12] test cov --- Cargo.lock | 8 + forester/Cargo.toml | 2 + forester/src/processor/v1/helpers.rs | 27 +- forester/src/processor/v1/tx_builder.rs | 344 +++++++++++++++++- .../tests/compact_nullify_regression.rs | 240 ++++++++++++ .../src/account_compression_cpi/nullify.rs | 71 +++- .../src/account_compression_cpi/sdk.rs | 127 ++++++- programs/registry/src/errors.rs | 4 + 8 files changed, 810 insertions(+), 13 deletions(-) create mode 100644 program-tests/registry-test/tests/compact_nullify_regression.rs diff --git a/Cargo.lock b/Cargo.lock index 9e293fd2b8..b5710c3afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2439,6 +2439,7 @@ dependencies = [ "async-channel 2.5.0", "async-trait", "base64 0.13.1", + "bincode", "borsh 0.10.4", "bs58", "clap 4.5.60", @@ -2472,6 +2473,7 @@ dependencies = [ "light-token", "light-token-client", "light-token-interface", + "mwmatching", "photon-api", "prometheus", "rand 0.8.5", @@ -4836,6 +4838,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "mwmatching" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f13b50448d988736cc2c938a76ae336241fcb31a225017c0e3121bd349e7dc06" + [[package]] name = "native-tls" version = "0.2.18" diff --git a/forester/Cargo.toml b/forester/Cargo.toml index 32d1df4e0d..8443848391 100644 --- a/forester/Cargo.toml +++ b/forester/Cargo.toml @@ -43,6 +43,7 @@ reqwest = { workspace = true, features = ["json", "rustls-tls", "blocking"] } futures = { workspace = true } thiserror = { workspace = true } borsh = { workspace = true } +bincode = "1.3" bs58 = { workspace = true } hex = { workspace = true } env_logger = { workspace = true } @@ -61,6 +62,7 @@ itertools = "0.14" async-channel = "2.5" solana-pubkey = { workspace = true } dotenvy = "0.15" +mwmatching = "0.1.1" [dev-dependencies] serial_test = { workspace = true } diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index d980b02e32..937c7ea42b 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -11,7 +11,7 @@ use forester_utils::{rpc_pool::SolanaRpcPool, utils::wait_for_indexer}; use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::TreeType; use light_registry::account_compression_cpi::sdk::{ - create_nullify_instruction, create_update_address_merkle_tree_instruction, + create_nullify_with_proof_accounts_instruction, create_update_address_merkle_tree_instruction, CreateNullifyInstructionInputs, UpdateAddressMerkleTreeInstructionInputs, }; use solana_program::instruction::Instruction; @@ -32,6 +32,19 @@ use crate::{ errors::ForesterError, }; +#[derive(Clone, Debug)] +pub enum PreparedV1Instruction { + AddressUpdate(Instruction), + StateNullify(StateNullifyInstruction), +} + +#[derive(Clone, Debug)] +pub struct StateNullifyInstruction { + pub instruction: Instruction, + pub proof_nodes: Vec<[u8; 32]>, + pub leaf_index: u64, +} + /// Work items should be of only one type and tree pub async fn fetch_proofs_and_create_instructions( authority: Pubkey, @@ -39,7 +52,7 @@ pub async fn fetch_proofs_and_create_instructions( pool: Arc>, epoch: u64, work_items: &[WorkItem], -) -> crate::Result<(Vec, Vec)> { +) -> crate::Result<(Vec, Vec)> { let mut proofs = Vec::new(); let mut instructions = vec![]; @@ -360,7 +373,7 @@ pub async fn fetch_proofs_and_create_instructions( }, epoch, ); - instructions.push(instruction); + instructions.push(PreparedV1Instruction::AddressUpdate(instruction)); } // Process state proofs and create instructions @@ -375,7 +388,7 @@ pub async fn fetch_proofs_and_create_instructions( for (item, proof) in state_items.iter().zip(state_proofs.into_iter()) { proofs.push(MerkleProofType::StateProof(proof.clone())); - let instruction = create_nullify_instruction( + let instruction = create_nullify_with_proof_accounts_instruction( CreateNullifyInstructionInputs { nullifier_queue: item.tree_account.queue, merkle_tree: item.tree_account.merkle_tree, @@ -389,7 +402,11 @@ pub async fn fetch_proofs_and_create_instructions( }, epoch, ); - instructions.push(instruction); + instructions.push(PreparedV1Instruction::StateNullify(StateNullifyInstruction { + instruction, + proof_nodes: proof.proof, + leaf_index: proof.leaf_index, + })); } Ok((proofs, instructions)) diff --git a/forester/src/processor/v1/tx_builder.rs b/forester/src/processor/v1/tx_builder.rs index 463cc0b2bf..a269d1d557 100644 --- a/forester/src/processor/v1/tx_builder.rs +++ b/forester/src/processor/v1/tx_builder.rs @@ -16,11 +16,22 @@ use crate::{ epoch_manager::WorkItem, processor::{ tx_cache::ProcessedHashCache, - v1::{config::BuildTransactionBatchConfig, helpers::fetch_proofs_and_create_instructions}, + v1::{ + config::BuildTransactionBatchConfig, + helpers::{ + fetch_proofs_and_create_instructions, PreparedV1Instruction, StateNullifyInstruction, + }, + }, }, smart_transaction::{create_smart_transaction, CreateSmartTransactionConfig}, Result, }; +use bincode::serialized_size; +use mwmatching::{Matching, SENTINEL}; + +const MAX_PAIRING_INSTRUCTIONS: usize = 96; +const MAX_PAIR_CANDIDATES: usize = 2_000; +const MIN_REMAINING_BLOCKS_FOR_PAIRING: u64 = 25; #[async_trait] #[allow(clippy::too_many_arguments)] @@ -58,6 +69,52 @@ impl EpochManagerTransactions { processed_hash_cache: cache, } } + + async fn should_attempt_pairing( + &self, + last_valid_block_height: u64, + state_nullify_count: usize, + ) -> bool { + let pair_candidates = pairing_candidate_count(state_nullify_count); + if !pairing_precheck_passes(state_nullify_count, pair_candidates) { + warn!( + "Skipping nullify pairing due to candidate explosion: count={}, pair_candidates={}", + state_nullify_count, pair_candidates + ); + return false; + } + + let conn = match self.pool.get_connection().await { + Ok(conn) => conn, + Err(e) => { + warn!( + "Skipping nullify pairing because RPC connection unavailable for block-height check: {}", + e + ); + return false; + } + }; + let current_block_height = match conn.get_block_height().await { + Ok(height) => height, + Err(e) => { + warn!( + "Skipping nullify pairing because block-height check failed: {}", + e + ); + return false; + } + }; + let remaining_blocks = last_valid_block_height.saturating_sub(current_block_height); + if !remaining_blocks_allows_pairing(remaining_blocks) { + warn!( + "Skipping nullify pairing near blockhash expiry: remaining_blocks={}", + remaining_blocks + ); + return false; + } + + true + } } #[async_trait] @@ -116,7 +173,7 @@ impl TransactionBuilder for EpochManagerTransactions { .collect::>(); let mut transactions = vec![]; - let all_instructions = match fetch_proofs_and_create_instructions( + let prepared_instructions = match fetch_proofs_and_create_instructions( payer.pubkey(), *derivation, self.pool.clone(), @@ -143,11 +200,32 @@ impl TransactionBuilder for EpochManagerTransactions { }; let batch_size = config.batch_size.max(1) as usize; + let state_nullify_count = prepared_instructions + .iter() + .filter(|ix| matches!(ix, PreparedV1Instruction::StateNullify(_))) + .count(); + let allow_pairing = if batch_size >= 2 { + self.should_attempt_pairing(last_valid_block_height, state_nullify_count) + .await + } else { + false + }; + let instruction_batches = build_instruction_batches( + prepared_instructions, + batch_size, + allow_pairing, + payer, + recent_blockhash, + last_valid_block_height, + priority_fee, + config.compute_unit_limit, + ) + .await?; - for instruction_chunk in all_instructions.chunks(batch_size) { + for instruction_chunk in instruction_batches { let (transaction, _) = create_smart_transaction(CreateSmartTransactionConfig { payer: payer.insecure_clone(), - instructions: instruction_chunk.to_vec(), + instructions: instruction_chunk, recent_blockhash: *recent_blockhash, compute_unit_price: priority_fee, compute_unit_limit: config.compute_unit_limit, @@ -171,3 +249,261 @@ impl TransactionBuilder for EpochManagerTransactions { Ok((transactions, last_valid_block_height)) } } + +async fn build_instruction_batches( + prepared_instructions: Vec, + batch_size: usize, + allow_pairing: bool, + payer: &Keypair, + recent_blockhash: &Hash, + last_valid_block_height: u64, + priority_fee: Option, + compute_unit_limit: Option, +) -> Result>> { + let mut address_instructions = Vec::new(); + let mut state_nullify_instructions = Vec::new(); + for prepared in prepared_instructions { + match prepared { + PreparedV1Instruction::AddressUpdate(ix) => address_instructions.push(ix), + PreparedV1Instruction::StateNullify(ix) => state_nullify_instructions.push(ix), + } + } + + let mut batches = Vec::new(); + for chunk in address_instructions.chunks(batch_size) { + batches.push(chunk.to_vec()); + } + + if state_nullify_instructions.is_empty() { + return Ok(batches); + } + + let paired_batches = if batch_size >= 2 && allow_pairing { + pair_state_nullify_batches( + state_nullify_instructions, + payer, + recent_blockhash, + last_valid_block_height, + priority_fee, + compute_unit_limit, + ) + .await? + } else { + state_nullify_instructions + .into_iter() + .map(|ix| vec![ix.instruction]) + .collect() + }; + batches.extend(paired_batches); + Ok(batches) +} + +async fn pair_state_nullify_batches( + state_nullify_instructions: Vec, + payer: &Keypair, + recent_blockhash: &Hash, + last_valid_block_height: u64, + priority_fee: Option, + compute_unit_limit: Option, +) -> Result>> { + let n = state_nullify_instructions.len(); + if n < 2 { + return Ok(state_nullify_instructions + .into_iter() + .map(|ix| vec![ix.instruction]) + .collect()); + } + + let mut edges: Vec<(usize, usize, i32)> = Vec::new(); + for i in 0..n { + for j in (i + 1)..n { + if !pair_fits_transaction_size( + &state_nullify_instructions[i].instruction, + &state_nullify_instructions[j].instruction, + payer, + recent_blockhash, + last_valid_block_height, + priority_fee, + compute_unit_limit, + ) + .await? + { + continue; + } + let overlap = state_nullify_instructions[i] + .proof_nodes + .iter() + .filter(|node| state_nullify_instructions[j].proof_nodes.contains(node)) + .count() as i32; + // Prioritize pair count first, then maximize proof overlap. + let weight = 10_000 + overlap; + edges.push((i, j, weight)); + } + } + + if edges.is_empty() { + return Ok(state_nullify_instructions + .into_iter() + .map(|ix| vec![ix.instruction]) + .collect()); + } + + let mates = Matching::new(edges).max_cardinality().solve(); + let mut used = vec![false; n]; + let mut paired_batches: Vec<(u64, Vec)> = Vec::new(); + let mut single_batches: Vec<(u64, Vec)> = Vec::new(); + + for i in 0..n { + if used[i] { + continue; + } + let mate = mates.get(i).copied().unwrap_or(SENTINEL); + if mate != SENTINEL && mate > i && mate < n { + used[i] = true; + used[mate] = true; + let (left, right) = + if state_nullify_instructions[i].leaf_index <= state_nullify_instructions[mate].leaf_index + { + (i, mate) + } else { + (mate, i) + }; + let min_leaf = state_nullify_instructions[left].leaf_index; + paired_batches.push(( + min_leaf, + vec![ + state_nullify_instructions[left].instruction.clone(), + state_nullify_instructions[right].instruction.clone(), + ], + )); + } + } + + for i in 0..n { + if !used[i] { + single_batches.push(( + state_nullify_instructions[i].leaf_index, + vec![state_nullify_instructions[i].instruction.clone()], + )); + } + } + + paired_batches.sort_by_key(|(leaf, _)| *leaf); + single_batches.sort_by_key(|(leaf, _)| *leaf); + paired_batches.extend(single_batches); + Ok(paired_batches.into_iter().map(|(_, batch)| batch).collect()) +} + +fn pairing_candidate_count(n: usize) -> usize { + n.saturating_sub(1).saturating_mul(n) / 2 +} + +fn pairing_precheck_passes(state_nullify_count: usize, pair_candidates: usize) -> bool { + if state_nullify_count < 2 { + return false; + } + if state_nullify_count > MAX_PAIRING_INSTRUCTIONS { + return false; + } + pair_candidates <= MAX_PAIR_CANDIDATES +} + +fn remaining_blocks_allows_pairing(remaining_blocks: u64) -> bool { + remaining_blocks > MIN_REMAINING_BLOCKS_FOR_PAIRING +} + +async fn pair_fits_transaction_size( + ix_a: &solana_program::instruction::Instruction, + ix_b: &solana_program::instruction::Instruction, + payer: &Keypair, + recent_blockhash: &Hash, + last_valid_block_height: u64, + priority_fee: Option, + compute_unit_limit: Option, +) -> Result { + let (tx, _) = create_smart_transaction(CreateSmartTransactionConfig { + payer: payer.insecure_clone(), + instructions: vec![ix_a.clone(), ix_b.clone()], + recent_blockhash: *recent_blockhash, + compute_unit_price: priority_fee, + compute_unit_limit, + last_valid_block_height, + }) + .await?; + + let tx_bytes = serialized_size(&tx)? as usize; + Ok(tx_bytes <= 1232) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn max_matching_prioritizes_cardinality() { + let edges = vec![(0usize, 1usize, 10_100i32), (1usize, 2usize, 10_090i32)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs = mates + .iter() + .enumerate() + .filter_map(|(i, mate)| { + if *mate != SENTINEL && *mate > i { + Some((i, *mate)) + } else { + None + } + }) + .collect::>(); + assert_eq!(pairs.len(), 1); + } + + #[test] + fn max_matching_handles_disconnected_graph() { + let edges = vec![(0usize, 1usize, 10_010i32), (2usize, 3usize, 10_005i32)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let matched_vertices = mates.iter().filter(|mate| **mate != SENTINEL).count(); + assert_eq!(matched_vertices, 4); + } + + #[test] + fn max_matching_returns_unmatched_for_empty_edges() { + let mates = Matching::new(vec![]).max_cardinality().solve(); + assert!(mates.is_empty()); + } + + #[test] + fn pairing_candidate_count_matches_combination_formula() { + assert_eq!(pairing_candidate_count(0), 0); + assert_eq!(pairing_candidate_count(1), 0); + assert_eq!(pairing_candidate_count(2), 1); + assert_eq!(pairing_candidate_count(3), 3); + assert_eq!(pairing_candidate_count(10), 45); + } + + #[test] + fn pairing_precheck_enforces_instruction_and_candidate_limits() { + let max_count_by_candidate_limit = 63; // 63 * 62 / 2 = 1953 + assert!(!pairing_precheck_passes(1, pairing_candidate_count(1))); + assert!(pairing_precheck_passes(2, pairing_candidate_count(2))); + assert!(pairing_precheck_passes( + max_count_by_candidate_limit, + pairing_candidate_count(max_count_by_candidate_limit) + )); + assert!(!pairing_precheck_passes( + max_count_by_candidate_limit + 1, + pairing_candidate_count(max_count_by_candidate_limit + 1) + )); + assert!(!pairing_precheck_passes( + MAX_PAIRING_INSTRUCTIONS + 1, + pairing_candidate_count(MAX_PAIRING_INSTRUCTIONS + 1) + )); + assert!(!pairing_precheck_passes(90, MAX_PAIR_CANDIDATES + 1)); + } + + #[test] + fn remaining_blocks_guard_is_strictly_greater_than_threshold() { + assert!(!remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING - 1)); + assert!(!remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING)); + assert!(remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING + 1)); + } +} diff --git a/program-tests/registry-test/tests/compact_nullify_regression.rs b/program-tests/registry-test/tests/compact_nullify_regression.rs new file mode 100644 index 0000000000..6539a246c0 --- /dev/null +++ b/program-tests/registry-test/tests/compact_nullify_regression.rs @@ -0,0 +1,240 @@ +use account_compression::state::QueueAccount; +use anchor_lang::InstructionData; +use forester_utils::account_zero_copy::{get_concurrent_merkle_tree, get_hash_set}; +use light_compressed_account::TreeType; +use light_hasher::Poseidon; +use light_program_test::{ + program_test::LightProgramTest, + utils::assert::assert_rpc_error, + ProgramTestConfig, +}; +use light_registry::{ + account_compression_cpi::sdk::{ + create_nullify_instruction, create_nullify_with_proof_accounts_instruction, + CreateNullifyInstructionInputs, + }, + errors::RegistryError, +}; +use light_test_utils::{e2e_test_env::init_program_test_env, Rpc}; +use serial_test::serial; +use solana_sdk::{ + instruction::Instruction, + pubkey::Pubkey, + signature::{Keypair, Signer}, +}; + +#[serial] +#[tokio::test] +async fn test_compact_nullify_validation_and_success() { + let mut rpc = LightProgramTest::new(ProgramTestConfig::default_with_batched_trees(true)) + .await + .unwrap(); + rpc.indexer = None; + let env = rpc.test_accounts.clone(); + let forester = Keypair::new(); + rpc.airdrop_lamports(&forester.pubkey(), 1_000_000_000) + .await + .unwrap(); + let merkle_tree_keypair = Keypair::new(); + let nullifier_queue_keypair = Keypair::new(); + let cpi_context_keypair = Keypair::new(); + let (mut rpc, state_tree_bundle) = { + let mut e2e_env = init_program_test_env(rpc, &env, 50).await; + e2e_env.indexer.state_merkle_trees.clear(); + e2e_env.keypair_action_config.fee_assert = false; + e2e_env + .indexer + .add_state_merkle_tree( + &mut e2e_env.rpc, + &merkle_tree_keypair, + &nullifier_queue_keypair, + &cpi_context_keypair, + None, + Some(forester.pubkey()), + TreeType::StateV1, + ) + .await; + e2e_env + .compress_sol_deterministic(&forester, 1_000_000, None) + .await; + e2e_env + .transfer_sol_deterministic(&forester, &Pubkey::new_unique(), None) + .await + .unwrap(); + (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) + }; + + let nullifier_queue = + unsafe { get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await } + .unwrap(); + let mut queue_index = None; + let mut account_hash = None; + for i in 0..nullifier_queue.get_capacity() { + let bucket = nullifier_queue.get_bucket(i).unwrap(); + if let Some(bucket) = bucket { + if bucket.sequence_number.is_none() { + queue_index = Some(i as u16); + account_hash = Some(bucket.value_bytes()); + break; + } + } + } + let queue_index = queue_index.unwrap(); + let account_hash = account_hash.unwrap(); + let leaf_index = state_tree_bundle + .merkle_tree + .get_leaf_index(&account_hash) + .unwrap() as u64; + let proof = state_tree_bundle + .merkle_tree + .get_proof_of_leaf(leaf_index as usize, false) + .unwrap(); + let proof_depth = proof.len(); + let onchain_tree = get_concurrent_merkle_tree::( + &mut rpc, + state_tree_bundle.accounts.merkle_tree, + ) + .await + .unwrap(); + let change_log_index = onchain_tree.changelog_index() as u64; + + let valid_ix = create_nullify_with_proof_accounts_instruction( + CreateNullifyInstructionInputs { + authority: forester.pubkey(), + nullifier_queue: state_tree_bundle.accounts.nullifier_queue, + merkle_tree: state_tree_bundle.accounts.merkle_tree, + change_log_indices: vec![change_log_index], + leaves_queue_indices: vec![queue_index], + indices: vec![leaf_index], + proofs: vec![proof], + derivation: forester.pubkey(), + is_metadata_forester: true, + }, + 0, + ); + + let mut empty_proof_accounts_ix = valid_ix.clone(); + empty_proof_accounts_ix + .accounts + .truncate(empty_proof_accounts_ix.accounts.len() - proof_depth); + let result = rpc + .create_and_send_transaction( + &[empty_proof_accounts_ix], + &forester.pubkey(), + &[&forester], + ) + .await; + assert_rpc_error(result, 0, RegistryError::EmptyProofAccounts.into()).unwrap(); + + let malformed_ix = Instruction { + program_id: light_registry::ID, + accounts: valid_ix.accounts.clone(), + data: light_registry::instruction::NullifyWithProofAccounts { + bump: 255, + change_log_indices: vec![change_log_index, change_log_index + 1], + leaves_queue_indices: vec![queue_index], + indices: vec![leaf_index], + } + .data(), + }; + let result = rpc + .create_and_send_transaction(&[malformed_ix], &forester.pubkey(), &[&forester]) + .await; + assert_rpc_error(result, 0, RegistryError::InvalidCompactNullifyInputs.into()).unwrap(); + + rpc.create_and_send_transaction(&[valid_ix], &forester.pubkey(), &[&forester]) + .await + .unwrap(); +} + +#[serial] +#[tokio::test] +async fn test_legacy_nullify_still_succeeds() { + let mut rpc = LightProgramTest::new(ProgramTestConfig::default_with_batched_trees(true)) + .await + .unwrap(); + rpc.indexer = None; + let env = rpc.test_accounts.clone(); + let forester = Keypair::new(); + rpc.airdrop_lamports(&forester.pubkey(), 1_000_000_000) + .await + .unwrap(); + let merkle_tree_keypair = Keypair::new(); + let nullifier_queue_keypair = Keypair::new(); + let cpi_context_keypair = Keypair::new(); + let (mut rpc, state_tree_bundle) = { + let mut e2e_env = init_program_test_env(rpc, &env, 50).await; + e2e_env.indexer.state_merkle_trees.clear(); + e2e_env.keypair_action_config.fee_assert = false; + e2e_env + .indexer + .add_state_merkle_tree( + &mut e2e_env.rpc, + &merkle_tree_keypair, + &nullifier_queue_keypair, + &cpi_context_keypair, + None, + Some(forester.pubkey()), + TreeType::StateV1, + ) + .await; + e2e_env + .compress_sol_deterministic(&forester, 1_000_000, None) + .await; + e2e_env + .transfer_sol_deterministic(&forester, &Pubkey::new_unique(), None) + .await + .unwrap(); + (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) + }; + let nullifier_queue = + unsafe { get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await } + .unwrap(); + let mut queue_index = None; + let mut account_hash = None; + for i in 0..nullifier_queue.get_capacity() { + let bucket = nullifier_queue.get_bucket(i).unwrap(); + if let Some(bucket) = bucket { + if bucket.sequence_number.is_none() { + queue_index = Some(i as u16); + account_hash = Some(bucket.value_bytes()); + break; + } + } + } + let queue_index = queue_index.unwrap(); + let account_hash = account_hash.unwrap(); + let leaf_index = state_tree_bundle + .merkle_tree + .get_leaf_index(&account_hash) + .unwrap() as u64; + let proof = state_tree_bundle + .merkle_tree + .get_proof_of_leaf(leaf_index as usize, false) + .unwrap(); + let onchain_tree = get_concurrent_merkle_tree::( + &mut rpc, + state_tree_bundle.accounts.merkle_tree, + ) + .await + .unwrap(); + let change_log_index = onchain_tree.changelog_index() as u64; + + let legacy_ix = create_nullify_instruction( + CreateNullifyInstructionInputs { + authority: forester.pubkey(), + nullifier_queue: state_tree_bundle.accounts.nullifier_queue, + merkle_tree: state_tree_bundle.accounts.merkle_tree, + change_log_indices: vec![change_log_index], + leaves_queue_indices: vec![queue_index], + indices: vec![leaf_index], + proofs: vec![proof], + derivation: forester.pubkey(), + is_metadata_forester: true, + }, + 0, + ); + rpc.create_and_send_transaction(&[legacy_ix], &forester.pubkey(), &[&forester]) + .await + .unwrap(); +} diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index badf4fdf7f..f4512d3f1a 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -5,6 +5,8 @@ use anchor_lang::prelude::*; use crate::{epoch::register_epoch::ForesterEpochPda, errors::RegistryError}; +const COMPACT_NULLIFY_PROOF_ACCOUNTS_LEN: usize = 16; + #[derive(Accounts)] pub struct NullifyLeaves<'info> { /// CHECK: only eligible foresters can nullify leaves. Is checked in ix. @@ -69,9 +71,12 @@ pub fn process_nullify_from_remaining_accounts( leaves_queue_indices: Vec, indices: Vec, ) -> Result<()> { - if ctx.remaining_accounts.is_empty() { - return err!(RegistryError::EmptyProofAccounts); - } + validate_compact_nullify_inputs( + &change_log_indices, + &leaves_queue_indices, + &indices, + ctx.remaining_accounts.len(), + )?; let proof_nodes: Vec<[u8; 32]> = ctx .remaining_accounts @@ -88,3 +93,63 @@ pub fn process_nullify_from_remaining_accounts( vec![proof_nodes], ) } + +pub(crate) fn validate_compact_nullify_inputs( + change_log_indices: &[u64], + leaves_queue_indices: &[u16], + indices: &[u64], + proof_accounts_len: usize, +) -> Result<()> { + if change_log_indices.len() != 1 + || leaves_queue_indices.len() != 1 + || indices.len() != 1 + { + return err!(RegistryError::InvalidCompactNullifyInputs); + } + if proof_accounts_len == 0 { + return err!(RegistryError::EmptyProofAccounts); + } + if proof_accounts_len != COMPACT_NULLIFY_PROOF_ACCOUNTS_LEN { + return err!(RegistryError::InvalidProofAccountsLength); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::validate_compact_nullify_inputs; + use crate::errors::RegistryError; + + #[test] + fn compact_nullify_inputs_validate_happy_path() { + let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 16); + assert!(result.is_ok()); + } + + #[test] + fn compact_nullify_inputs_reject_empty_proof_accounts() { + let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 0); + assert_eq!( + result.err().unwrap(), + RegistryError::EmptyProofAccounts.into() + ); + } + + #[test] + fn compact_nullify_inputs_reject_vector_length_mismatch() { + let result = validate_compact_nullify_inputs(&[1, 2], &[1], &[42], 16); + assert_eq!( + result.err().unwrap(), + RegistryError::InvalidCompactNullifyInputs.into() + ); + } + + #[test] + fn compact_nullify_inputs_reject_invalid_proof_accounts_length() { + let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 15); + assert_eq!( + result.err().unwrap(), + RegistryError::InvalidProofAccountsLength.into() + ); + } +} diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 057dcbf7ef..3ef001ce77 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -29,6 +29,42 @@ pub struct CreateNullifyInstructionInputs { pub fn create_nullify_instruction( inputs: CreateNullifyInstructionInputs, epoch: u64, +) -> Instruction { + let register_program_pda = get_registered_program_pda(&crate::ID); + let registered_forester_pda = if inputs.is_metadata_forester { + None + } else { + Some(get_forester_epoch_pda_from_authority(&inputs.derivation, epoch).0) + }; + let (cpi_authority, bump) = get_cpi_authority_pda(); + let instruction_data = crate::instruction::Nullify { + bump, + change_log_indices: inputs.change_log_indices, + leaves_queue_indices: inputs.leaves_queue_indices, + indices: inputs.indices, + proofs: inputs.proofs, + }; + + let accounts = crate::accounts::NullifyLeaves { + authority: inputs.authority, + registered_forester_pda, + registered_program_pda: register_program_pda, + nullifier_queue: inputs.nullifier_queue, + merkle_tree: inputs.merkle_tree, + log_wrapper: NOOP_PUBKEY.into(), + cpi_authority, + account_compression_program: account_compression::ID, + }; + Instruction { + program_id: crate::ID, + accounts: accounts.to_account_metas(Some(true)), + data: instruction_data.data(), + } +} + +pub fn create_nullify_with_proof_accounts_instruction( + inputs: CreateNullifyInstructionInputs, + epoch: u64, ) -> Instruction { let register_program_pda = get_registered_program_pda(&crate::ID); let registered_forester_pda = if inputs.is_metadata_forester { @@ -57,7 +93,10 @@ pub fn create_nullify_instruction( let mut accounts = base_accounts.to_account_metas(Some(true)); for proof in inputs.proofs { for node in proof { - accounts.push(AccountMeta::new_readonly(Pubkey::new_from_array(node), false)); + accounts.push(AccountMeta::new_readonly( + Pubkey::new_from_array(node), + false, + )); } } @@ -551,3 +590,89 @@ pub fn create_rollover_batch_address_tree_instruction( data: instruction_data.data(), } } + +#[cfg(test)] +mod tests { + use anchor_lang::Discriminator; + + use super::*; + + #[test] + fn create_nullify_instruction_uses_legacy_payload() { + let authority = Pubkey::new_unique(); + let derivation = Pubkey::new_unique(); + let nullifier_queue = Pubkey::new_unique(); + let merkle_tree = Pubkey::new_unique(); + let proof = (0..16) + .map(|i| { + let mut node = [0u8; 32]; + node[0] = i as u8; + node + }) + .collect::>(); + let ix = create_nullify_instruction( + CreateNullifyInstructionInputs { + authority, + nullifier_queue, + merkle_tree, + change_log_indices: vec![7], + leaves_queue_indices: vec![11], + indices: vec![42], + proofs: vec![proof], + derivation, + is_metadata_forester: false, + }, + 1, + ); + + assert_eq!(ix.program_id, crate::ID); + assert_eq!(ix.accounts.len(), 8); + assert_eq!(&ix.data[..8], crate::instruction::Nullify::DISCRIMINATOR); + assert_eq!(ix.data.len(), 559); + } + + #[test] + fn create_nullify_with_proof_accounts_instruction_uses_compact_payload_and_remaining_accounts() + { + let authority = Pubkey::new_unique(); + let derivation = Pubkey::new_unique(); + let nullifier_queue = Pubkey::new_unique(); + let merkle_tree = Pubkey::new_unique(); + let proof = (0..16) + .map(|i| { + let mut node = [0u8; 32]; + node[0] = i as u8; + node + }) + .collect::>(); + let ix = create_nullify_with_proof_accounts_instruction( + CreateNullifyInstructionInputs { + authority, + nullifier_queue, + merkle_tree, + change_log_indices: vec![7], + leaves_queue_indices: vec![11], + indices: vec![42], + proofs: vec![proof.clone()], + derivation, + is_metadata_forester: false, + }, + 1, + ); + + assert_eq!(ix.program_id, crate::ID); + assert_eq!(ix.accounts.len(), 8 + 16); + for (account_meta, node) in ix.accounts[8..].iter().zip(proof.iter()) { + assert_eq!(account_meta.pubkey, Pubkey::new_from_array(*node)); + assert!(!account_meta.is_signer); + assert!(!account_meta.is_writable); + } + + assert_eq!( + &ix.data[..8], + crate::instruction::NullifyWithProofAccounts::DISCRIMINATOR + ); + // 8-byte discriminator + 31-byte compact payload. + assert_eq!(ix.data.len(), 39); + } +} diff --git a/programs/registry/src/errors.rs b/programs/registry/src/errors.rs index abe4318fb9..41cfb9987c 100644 --- a/programs/registry/src/errors.rs +++ b/programs/registry/src/errors.rs @@ -40,6 +40,10 @@ pub enum RegistryError { EmptyIndices, #[msg("Proof accounts cannot be empty")] EmptyProofAccounts, + #[msg("Compact nullify proof accounts length is invalid")] + InvalidProofAccountsLength, + #[msg("Compact nullify supports exactly one change, queue index, and leaf index")] + InvalidCompactNullifyInputs, #[msg("Failed to borrow account data")] BorrowAccountDataFailed, #[msg("Failed to serialize instruction data")] From 11a89fc2e4d233c1410123a971b07cf4b702f5b4 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Sun, 15 Mar 2026 19:12:23 +0000 Subject: [PATCH 03/12] wip --- forester/src/processor/v1/send_transaction.rs | 169 ++++++++++++++---- 1 file changed, 131 insertions(+), 38 deletions(-) diff --git a/forester/src/processor/v1/send_transaction.rs b/forester/src/processor/v1/send_transaction.rs index b5282bc47a..b19bf77fbc 100644 --- a/forester/src/processor/v1/send_transaction.rs +++ b/forester/src/processor/v1/send_transaction.rs @@ -35,8 +35,6 @@ use crate::{ struct PreparedBatchData { work_items: Vec, - recent_blockhash: Hash, - last_valid_block_height: u64, priority_fee: Option, timeout_deadline: Instant, } @@ -115,11 +113,9 @@ pub async fn send_batched_transactions BLOCKHASH_REFRESH_INTERVAL { - match pool.get_connection().await { - Ok(mut rpc) => match rpc.get_latest_blockhash().await { - Ok((new_hash, new_height)) => { - recent_blockhash = new_hash; - last_valid_block_height = new_height; - last_blockhash_refresh = Instant::now(); - debug!(tree = %tree_accounts.merkle_tree, "Refreshed blockhash"); - } - Err(e) => { - warn!(tree = %tree_accounts.merkle_tree, "Failed to refresh blockhash: {:?}", e); - } - }, - Err(e) => { - warn!(tree = %tree_accounts.merkle_tree, "Failed to get RPC for blockhash refresh: {:?}", e); - } + match fetch_latest_blockhash(&pool, &tree_id_str).await { + Ok((new_hash, new_height)) => { + recent_blockhash = new_hash; + last_valid_block_height = new_height; + debug!(tree = %tree_accounts.merkle_tree, "Fetched fresh blockhash for chunk build"); + } + Err(e) => { + warn!( + tree = %tree_accounts.merkle_tree, + "Failed to fetch fresh blockhash for chunk build, using last known value: {:?}", + e + ); } } trace!(tree = %tree_accounts.merkle_tree, "Processing chunk of size {}", work_chunk.len()); let build_start_time = Instant::now(); - let (transactions_to_send, chunk_last_valid_block_height) = match transaction_builder + let (mut transactions_to_send, chunk_last_valid_block_height) = match transaction_builder .build_signed_transaction_batch( payer, derivation, @@ -184,6 +175,28 @@ pub async fn send_batched_transactions { + if let Err(e) = resign_transactions(&mut transactions_to_send, payer, send_blockhash) { + warn!( + tree = %tree_accounts.merkle_tree, + "Failed to re-sign chunk with freshest blockhash, skipping chunk: {:?}", + e + ); + continue; + } + send_last_valid_block_height = send_last_valid; + } + Err(e) => { + warn!( + tree = %tree_accounts.merkle_tree, + "Failed to fetch fresh blockhash before send; using build-time blockhash: {:?}", + e + ); + } + } + let send_context = ChunkSendContext { pool: Arc::clone(&pool), max_concurrent_sends: effective_max_concurrent_sends, @@ -198,7 +211,7 @@ pub async fn send_batched_transactions( return Ok(None); // Return None to indicate no work } - let (recent_blockhash, last_valid_block_height, priority_fee) = { - let mut rpc = pool.get_connection().await.map_err(|e| { + let priority_fee = { + let rpc = pool.get_connection().await.map_err(|e| { error!( tree = %tree_id_str, - "Failed to get RPC for blockhash/priority fee: {:?}", + "Failed to get RPC for priority fee: {:?}", e ); ForesterError::RpcPool(e) })?; - let r_blockhash = rpc.get_latest_blockhash().await.map_err(|e| { - error!(tree = %tree_id_str, "Failed to get latest blockhash: {:?}", e); - ForesterError::Rpc(e) - })?; let forester_epoch_pda_pubkey = get_forester_epoch_pda_from_authority(derivation, transaction_builder.epoch()).0; let account_keys = vec![ @@ -290,13 +299,12 @@ async fn prepare_batch_prerequisites( tree_accounts.queue, tree_accounts.merkle_tree, ]; - let priority_fee = PriorityFeeConfig { + PriorityFeeConfig { compute_unit_price: config.build_transaction_batch_config.compute_unit_price, enable_priority_fees: config.build_transaction_batch_config.enable_priority_fees, } .resolve(&*rpc, account_keys) - .await?; - (r_blockhash.0, r_blockhash.1, priority_fee) + .await? }; let work_items: Vec = queue_item_data @@ -311,13 +319,98 @@ async fn prepare_batch_prerequisites( Ok(Some(PreparedBatchData { work_items, - recent_blockhash, - last_valid_block_height, priority_fee, timeout_deadline, })) } +async fn fetch_latest_blockhash( + pool: &Arc>, + tree_id_str: &str, +) -> std::result::Result<(Hash, u64), ForesterError> { + let mut rpc = pool.get_connection().await.map_err(|e| { + error!( + tree = %tree_id_str, + "Failed to get RPC for blockhash fetch: {:?}", + e + ); + ForesterError::RpcPool(e) + })?; + rpc.get_latest_blockhash().await.map_err(|e| { + error!(tree = %tree_id_str, "Failed to get latest blockhash: {:?}", e); + ForesterError::Rpc(e) + }) +} + +fn resign_transactions( + transactions: &mut [Transaction], + payer: &Keypair, + recent_blockhash: Hash, +) -> std::result::Result<(), ForesterError> { + for tx in transactions.iter_mut() { + tx.try_sign(&[payer], recent_blockhash) + .map_err(|e| ForesterError::General { + error: format!("failed to re-sign transaction: {}", e), + })?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::resign_transactions; + use crate::errors::ForesterError; + use solana_sdk::{ + hash::Hash, + instruction::{AccountMeta, Instruction}, + pubkey::Pubkey, + signature::{Keypair, Signer}, + transaction::Transaction, + }; + + #[test] + fn resign_transactions_updates_signature_with_new_blockhash() { + let payer = Keypair::new(); + let ix = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![AccountMeta::new(payer.pubkey(), true)], + data: vec![], + }; + let mut tx = Transaction::new_with_payer(&[ix], Some(&payer.pubkey())); + let initial_hash = Hash::new_unique(); + tx.try_sign(&[&payer], initial_hash).unwrap(); + let old_signature = tx.signatures[0]; + + let mut txs = vec![tx]; + resign_transactions(&mut txs, &payer, Hash::new_unique()).unwrap(); + + assert_ne!(txs[0].signatures[0], old_signature); + } + + #[test] + fn resign_transactions_fails_when_extra_signer_is_required() { + let payer = Keypair::new(); + let extra_signer = Keypair::new(); + let ix = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![ + AccountMeta::new(payer.pubkey(), true), + AccountMeta::new_readonly(extra_signer.pubkey(), true), + ], + data: vec![], + }; + let mut tx = Transaction::new_with_payer(&[ix], Some(&payer.pubkey())); + tx.try_sign(&[&payer, &extra_signer], Hash::new_unique()) + .unwrap(); + + let mut txs = vec![tx]; + let err = resign_transactions(&mut txs, &payer, Hash::new_unique()) + .expect_err("re-sign should fail when required signer is missing"); + + assert!(matches!(err, ForesterError::General { .. })); + } +} + fn compute_effective_max_concurrent_sends( config: &SendBatchedTransactionsConfig, configured_max: usize, From c94812ee2470eba079d0ab7ba6846d23271367b5 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 14:59:35 +0000 Subject: [PATCH 04/12] rm re-sign --- forester/src/processor/v1/send_transaction.rs | 140 ++++-------------- 1 file changed, 30 insertions(+), 110 deletions(-) diff --git a/forester/src/processor/v1/send_transaction.rs b/forester/src/processor/v1/send_transaction.rs index b19bf77fbc..b5990baef1 100644 --- a/forester/src/processor/v1/send_transaction.rs +++ b/forester/src/processor/v1/send_transaction.rs @@ -35,6 +35,8 @@ use crate::{ struct PreparedBatchData { work_items: Vec, + recent_blockhash: Hash, + last_valid_block_height: u64, priority_fee: Option, timeout_deadline: Instant, } @@ -114,8 +116,11 @@ pub async fn send_batched_transactions { - recent_blockhash = new_hash; - last_valid_block_height = new_height; - debug!(tree = %tree_accounts.merkle_tree, "Fetched fresh blockhash for chunk build"); - } - Err(e) => { - warn!( - tree = %tree_accounts.merkle_tree, - "Failed to fetch fresh blockhash for chunk build, using last known value: {:?}", - e - ); + if last_blockhash_refresh.elapsed() > BLOCKHASH_REFRESH_INTERVAL { + match fetch_latest_blockhash(&pool, &tree_id_str).await { + Ok((new_hash, new_height)) => { + recent_blockhash = new_hash; + last_valid_block_height = new_height; + last_blockhash_refresh = Instant::now(); + debug!(tree = %tree_accounts.merkle_tree, "Refreshed blockhash"); + } + Err(e) => { + warn!(tree = %tree_accounts.merkle_tree, "Failed to refresh blockhash: {:?}", e); + } } } trace!(tree = %tree_accounts.merkle_tree, "Processing chunk of size {}", work_chunk.len()); let build_start_time = Instant::now(); - let (mut transactions_to_send, chunk_last_valid_block_height) = match transaction_builder + let (transactions_to_send, chunk_last_valid_block_height) = match transaction_builder .build_signed_transaction_batch( payer, derivation, @@ -175,28 +179,6 @@ pub async fn send_batched_transactions { - if let Err(e) = resign_transactions(&mut transactions_to_send, payer, send_blockhash) { - warn!( - tree = %tree_accounts.merkle_tree, - "Failed to re-sign chunk with freshest blockhash, skipping chunk: {:?}", - e - ); - continue; - } - send_last_valid_block_height = send_last_valid; - } - Err(e) => { - warn!( - tree = %tree_accounts.merkle_tree, - "Failed to fetch fresh blockhash before send; using build-time blockhash: {:?}", - e - ); - } - } - let send_context = ChunkSendContext { pool: Arc::clone(&pool), max_concurrent_sends: effective_max_concurrent_sends, @@ -211,7 +193,7 @@ pub async fn send_batched_transactions( return Ok(None); // Return None to indicate no work } - let priority_fee = { + let (priority_fee, recent_blockhash, last_valid_block_height) = { let rpc = pool.get_connection().await.map_err(|e| { error!( tree = %tree_id_str, @@ -299,12 +281,17 @@ async fn prepare_batch_prerequisites( tree_accounts.queue, tree_accounts.merkle_tree, ]; - PriorityFeeConfig { + let priority_fee = PriorityFeeConfig { compute_unit_price: config.build_transaction_batch_config.compute_unit_price, enable_priority_fees: config.build_transaction_batch_config.enable_priority_fees, } .resolve(&*rpc, account_keys) - .await? + .await?; + + let (recent_blockhash, last_valid_block_height) = + fetch_latest_blockhash(pool, &tree_id_str).await?; + + (priority_fee, recent_blockhash, last_valid_block_height) }; let work_items: Vec = queue_item_data @@ -319,6 +306,8 @@ async fn prepare_batch_prerequisites( Ok(Some(PreparedBatchData { work_items, + recent_blockhash, + last_valid_block_height, priority_fee, timeout_deadline, })) @@ -342,75 +331,6 @@ async fn fetch_latest_blockhash( }) } -fn resign_transactions( - transactions: &mut [Transaction], - payer: &Keypair, - recent_blockhash: Hash, -) -> std::result::Result<(), ForesterError> { - for tx in transactions.iter_mut() { - tx.try_sign(&[payer], recent_blockhash) - .map_err(|e| ForesterError::General { - error: format!("failed to re-sign transaction: {}", e), - })?; - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::resign_transactions; - use crate::errors::ForesterError; - use solana_sdk::{ - hash::Hash, - instruction::{AccountMeta, Instruction}, - pubkey::Pubkey, - signature::{Keypair, Signer}, - transaction::Transaction, - }; - - #[test] - fn resign_transactions_updates_signature_with_new_blockhash() { - let payer = Keypair::new(); - let ix = Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![AccountMeta::new(payer.pubkey(), true)], - data: vec![], - }; - let mut tx = Transaction::new_with_payer(&[ix], Some(&payer.pubkey())); - let initial_hash = Hash::new_unique(); - tx.try_sign(&[&payer], initial_hash).unwrap(); - let old_signature = tx.signatures[0]; - - let mut txs = vec![tx]; - resign_transactions(&mut txs, &payer, Hash::new_unique()).unwrap(); - - assert_ne!(txs[0].signatures[0], old_signature); - } - - #[test] - fn resign_transactions_fails_when_extra_signer_is_required() { - let payer = Keypair::new(); - let extra_signer = Keypair::new(); - let ix = Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![ - AccountMeta::new(payer.pubkey(), true), - AccountMeta::new_readonly(extra_signer.pubkey(), true), - ], - data: vec![], - }; - let mut tx = Transaction::new_with_payer(&[ix], Some(&payer.pubkey())); - tx.try_sign(&[&payer, &extra_signer], Hash::new_unique()) - .unwrap(); - - let mut txs = vec![tx]; - let err = resign_transactions(&mut txs, &payer, Hash::new_unique()) - .expect_err("re-sign should fail when required signer is missing"); - - assert!(matches!(err, ForesterError::General { .. })); - } -} - fn compute_effective_max_concurrent_sends( config: &SendBatchedTransactionsConfig, configured_max: usize, From d6dbb450206d324588105dd64c301eabfeb37bc9 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:11:25 +0000 Subject: [PATCH 05/12] upd --- forester/docs/v1_forester_flows.md | 163 ++++++++++++++++++ forester/src/processor/v1/helpers.rs | 4 +- forester/src/processor/v1/send_transaction.rs | 61 +++---- .../tests/compact_nullify_regression.rs | 7 +- .../src/account_compression_cpi/nullify.rs | 43 +++-- .../src/account_compression_cpi/sdk.rs | 10 +- programs/registry/src/lib.rs | 4 +- 7 files changed, 223 insertions(+), 69 deletions(-) create mode 100644 forester/docs/v1_forester_flows.md diff --git a/forester/docs/v1_forester_flows.md b/forester/docs/v1_forester_flows.md new file mode 100644 index 0000000000..6f6b1d1d53 --- /dev/null +++ b/forester/docs/v1_forester_flows.md @@ -0,0 +1,163 @@ +# Forester V1 Flows (PR: Compact Nullify + Blockhash) + +## 1. Transaction Send Flow (Blockhash) + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ send_batched_transactions │ +└─────────────────────────────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────┐ + │ prepare_batch_prerequisites │ + │ - fetch queue items │ + │ - single RPC: blockhash + │ + │ priority_fee (same connection) │ + │ - PreparedBatchData: │ + │ recent_blockhash │ + │ last_valid_block_height │ + └──────────────┬───────────────────┘ + │ + ▼ + ┌──────────────────────────────────┐ + │ for each work_chunk (100 items) │ + └──────────────┬───────────────────┘ + │ + ┌────────────┴────────────┐ + │ elapsed > 30s? │ + │ YES → refresh blockhash│ + │ (pool.get_connection │ + │ → rpc.get_latest_ │ + │ blockhash) │ + │ NO → keep current │ + └────────────┬────────────┘ + │ + ▼ + ┌──────────────────────────────────┐ + │ build_signed_transaction_batch │ + │ (recent_blockhash, │ + │ last_valid_block_height) │ + │ → (txs, chunk_last_valid_ │ + │ block_height) │ + └──────────────┬───────────────────┘ + │ + ▼ + ┌──────────────────────────────────┐ + │ execute_transaction_chunk_sending │ + │ PreparedTransaction::legacy( │ + │ tx, chunk_last_valid_block_ │ + │ height) │ + │ - send + confirm │ + │ - blockhash expiry check via │ + │ last_valid_block_height │ + └──────────────────────────────────┘ + + No refetch-before-send. No re-sign. +``` + +## 2. State Nullify Instruction Flow (Compact vs Legacy) + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ Registry: nullify instruction paths │ +└─────────────────────────────────────────────────────────────────────────────────┘ + + LEGACY (proof in ix data) COMPACT (proof in remaining_accounts) + ─────────────────────── ──────────────────────────────────── + + create_nullify_instruction() create_nullify_with_proof_accounts_instruction() + │ │ + │ ix data: [change_log, queue_idx, │ ix data: [change_log, queue_idx, + │ leaf_idx, proofs[16][32]] │ leaf_idx] (no proofs) + │ │ + │ remaining_accounts: standard │ remaining_accounts: 16 proof + │ (authority, merkle_tree, queue...) │ account pubkeys (key = node bytes) + │ │ + ▼ ▼ + process_nullify() process_nullify_2() + (proofs from ix data) - validate: 1 change, 1 queue, 1 index + - validate: exactly 16 proof accounts + - extract_proof_nodes_from_remaining_accounts + - process_nullify(..., vec![proof_nodes]) + + Forester V1 uses COMPACT only (create_nullify_with_proof_accounts_instruction). +``` + +## 3. Forester V1 State Nullify Pairing Flow + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ build_instruction_batches (state nullify path) │ +└─────────────────────────────────────────────────────────────────────────────────┘ + + fetch_proofs_and_create_instructions + │ + │ For each state item: + │ create_nullify_with_proof_accounts_instruction (compact) + │ → StateNullifyInstruction { instruction, proof_nodes, leaf_index } + │ + ▼ + ┌─────────────────────────────────────────────────────────────────────────────┐ + │ allow_pairing? │ + │ batch_size >= 2 AND should_attempt_pairing() │ + └─────────────────────────────────────────────────────────────────────────────┘ + │ + │ should_attempt_pairing checks: + │ - pair_candidates = n*(n-1)/2 <= 2000 (MAX_PAIR_CANDIDATES) + │ - state_nullify_count <= 96 (MAX_PAIRING_INSTRUCTIONS) + │ - remaining_blocks = last_valid - current > 25 (MIN_REMAINING_BLOCKS_FOR_PAIRING) + │ + ├── NO → each nullify → 1 tx (no pairing) + │ + └── YES → pair_state_nullify_batches + │ + │ For each pair (i,j): + │ - pair_fits_transaction_size(ix_i, ix_j)? (serialized <= 1232) + │ - weight = 10000 + proof_overlap_count + │ + │ Max-cardinality matching (mwmatching) + │ - prioritize number of pairs + │ - then maximize proof overlap (fewer unique accounts) + │ + ▼ + Output: Vec> + - paired: [ix_a, ix_b] in one tx + - unpaired: [ix] in one tx + + Address updates: no pairing, chunked by batch_size only. +``` + +## 4. End-to-End Forester V1 State Tree Flow + +``` + Queue (state nullifier) Indexer (proofs) + │ │ + └──────────┬─────────────────┘ + │ + ▼ + prepare_batch_prerequisites + - queue items + - blockhash + last_valid_block_height + - priority_fee + │ + ▼ + for chunk in work_items.chunks(100): + refresh blockhash if 30s elapsed + │ + ▼ + build_signed_transaction_batch + │ + ├─ fetch_proofs_and_create_instructions + │ - state: compact nullify ix (proof in remaining_accounts) + │ - address: update ix + │ + ├─ build_instruction_batches + │ - address: chunk by batch_size + │ - state nullify: pair if allow_pairing else 1-per-tx + │ + └─ create_smart_transaction per batch + │ + ▼ + execute_transaction_chunk_sending + - PreparedTransaction::legacy(tx, chunk_last_valid_block_height) + - send + confirm with blockhash expiry check +``` diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index 937c7ea42b..6f37fa8128 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -11,7 +11,7 @@ use forester_utils::{rpc_pool::SolanaRpcPool, utils::wait_for_indexer}; use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::TreeType; use light_registry::account_compression_cpi::sdk::{ - create_nullify_with_proof_accounts_instruction, create_update_address_merkle_tree_instruction, + create_nullify_2_instruction, create_update_address_merkle_tree_instruction, CreateNullifyInstructionInputs, UpdateAddressMerkleTreeInstructionInputs, }; use solana_program::instruction::Instruction; @@ -388,7 +388,7 @@ pub async fn fetch_proofs_and_create_instructions( for (item, proof) in state_items.iter().zip(state_proofs.into_iter()) { proofs.push(MerkleProofType::StateProof(proof.clone())); - let instruction = create_nullify_with_proof_accounts_instruction( + let instruction = create_nullify_2_instruction( CreateNullifyInstructionInputs { nullifier_queue: item.tree_account.queue, merkle_tree: item.tree_account.merkle_tree, diff --git a/forester/src/processor/v1/send_transaction.rs b/forester/src/processor/v1/send_transaction.rs index b5990baef1..b5282bc47a 100644 --- a/forester/src/processor/v1/send_transaction.rs +++ b/forester/src/processor/v1/send_transaction.rs @@ -115,11 +115,10 @@ pub async fn send_batched_transactions BLOCKHASH_REFRESH_INTERVAL { - match fetch_latest_blockhash(&pool, &tree_id_str).await { - Ok((new_hash, new_height)) => { - recent_blockhash = new_hash; - last_valid_block_height = new_height; - last_blockhash_refresh = Instant::now(); - debug!(tree = %tree_accounts.merkle_tree, "Refreshed blockhash"); - } + match pool.get_connection().await { + Ok(mut rpc) => match rpc.get_latest_blockhash().await { + Ok((new_hash, new_height)) => { + recent_blockhash = new_hash; + last_valid_block_height = new_height; + last_blockhash_refresh = Instant::now(); + debug!(tree = %tree_accounts.merkle_tree, "Refreshed blockhash"); + } + Err(e) => { + warn!(tree = %tree_accounts.merkle_tree, "Failed to refresh blockhash: {:?}", e); + } + }, Err(e) => { - warn!(tree = %tree_accounts.merkle_tree, "Failed to refresh blockhash: {:?}", e); + warn!(tree = %tree_accounts.merkle_tree, "Failed to get RPC for blockhash refresh: {:?}", e); } } } @@ -264,15 +269,19 @@ async fn prepare_batch_prerequisites( return Ok(None); // Return None to indicate no work } - let (priority_fee, recent_blockhash, last_valid_block_height) = { - let rpc = pool.get_connection().await.map_err(|e| { + let (recent_blockhash, last_valid_block_height, priority_fee) = { + let mut rpc = pool.get_connection().await.map_err(|e| { error!( tree = %tree_id_str, - "Failed to get RPC for priority fee: {:?}", + "Failed to get RPC for blockhash/priority fee: {:?}", e ); ForesterError::RpcPool(e) })?; + let r_blockhash = rpc.get_latest_blockhash().await.map_err(|e| { + error!(tree = %tree_id_str, "Failed to get latest blockhash: {:?}", e); + ForesterError::Rpc(e) + })?; let forester_epoch_pda_pubkey = get_forester_epoch_pda_from_authority(derivation, transaction_builder.epoch()).0; let account_keys = vec![ @@ -287,11 +296,7 @@ async fn prepare_batch_prerequisites( } .resolve(&*rpc, account_keys) .await?; - - let (recent_blockhash, last_valid_block_height) = - fetch_latest_blockhash(pool, &tree_id_str).await?; - - (priority_fee, recent_blockhash, last_valid_block_height) + (r_blockhash.0, r_blockhash.1, priority_fee) }; let work_items: Vec = queue_item_data @@ -313,24 +318,6 @@ async fn prepare_batch_prerequisites( })) } -async fn fetch_latest_blockhash( - pool: &Arc>, - tree_id_str: &str, -) -> std::result::Result<(Hash, u64), ForesterError> { - let mut rpc = pool.get_connection().await.map_err(|e| { - error!( - tree = %tree_id_str, - "Failed to get RPC for blockhash fetch: {:?}", - e - ); - ForesterError::RpcPool(e) - })?; - rpc.get_latest_blockhash().await.map_err(|e| { - error!(tree = %tree_id_str, "Failed to get latest blockhash: {:?}", e); - ForesterError::Rpc(e) - }) -} - fn compute_effective_max_concurrent_sends( config: &SendBatchedTransactionsConfig, configured_max: usize, diff --git a/program-tests/registry-test/tests/compact_nullify_regression.rs b/program-tests/registry-test/tests/compact_nullify_regression.rs index 6539a246c0..93f7b83ff1 100644 --- a/program-tests/registry-test/tests/compact_nullify_regression.rs +++ b/program-tests/registry-test/tests/compact_nullify_regression.rs @@ -10,8 +10,7 @@ use light_program_test::{ }; use light_registry::{ account_compression_cpi::sdk::{ - create_nullify_instruction, create_nullify_with_proof_accounts_instruction, - CreateNullifyInstructionInputs, + create_nullify_2_instruction, create_nullify_instruction, CreateNullifyInstructionInputs, }, errors::RegistryError, }; @@ -98,7 +97,7 @@ async fn test_compact_nullify_validation_and_success() { .unwrap(); let change_log_index = onchain_tree.changelog_index() as u64; - let valid_ix = create_nullify_with_proof_accounts_instruction( + let valid_ix = create_nullify_2_instruction( CreateNullifyInstructionInputs { authority: forester.pubkey(), nullifier_queue: state_tree_bundle.accounts.nullifier_queue, @@ -129,7 +128,7 @@ async fn test_compact_nullify_validation_and_success() { let malformed_ix = Instruction { program_id: light_registry::ID, accounts: valid_ix.accounts.clone(), - data: light_registry::instruction::NullifyWithProofAccounts { + data: light_registry::instruction::Nullify2 { bump: 255, change_log_indices: vec![change_log_index, change_log_index + 1], leaves_queue_indices: vec![queue_index], diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index f4512d3f1a..0bad92ad45 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -5,7 +5,7 @@ use anchor_lang::prelude::*; use crate::{epoch::register_epoch::ForesterEpochPda, errors::RegistryError}; -const COMPACT_NULLIFY_PROOF_ACCOUNTS_LEN: usize = 16; +const NULLIFY_2_PROOF_ACCOUNTS_LEN: usize = 16; #[derive(Accounts)] pub struct NullifyLeaves<'info> { @@ -64,25 +64,21 @@ pub fn process_nullify( ) } -pub fn process_nullify_from_remaining_accounts( +pub fn process_nullify_2( ctx: &Context, bump: u8, change_log_indices: Vec, leaves_queue_indices: Vec, indices: Vec, ) -> Result<()> { - validate_compact_nullify_inputs( + validate_nullify_2_inputs( &change_log_indices, &leaves_queue_indices, &indices, ctx.remaining_accounts.len(), )?; - let proof_nodes: Vec<[u8; 32]> = ctx - .remaining_accounts - .iter() - .map(|account_info| account_info.key().to_bytes()) - .collect(); + let proof_nodes = extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); process_nullify( ctx, @@ -94,7 +90,16 @@ pub fn process_nullify_from_remaining_accounts( ) } -pub(crate) fn validate_compact_nullify_inputs( +fn extract_proof_nodes_from_remaining_accounts( + remaining_accounts: &[AccountInfo<'_>], +) -> Vec<[u8; 32]> { + remaining_accounts + .iter() + .map(|account_info| account_info.key().to_bytes()) + .collect() +} + +pub(crate) fn validate_nullify_2_inputs( change_log_indices: &[u64], leaves_queue_indices: &[u16], indices: &[u64], @@ -109,7 +114,7 @@ pub(crate) fn validate_compact_nullify_inputs( if proof_accounts_len == 0 { return err!(RegistryError::EmptyProofAccounts); } - if proof_accounts_len != COMPACT_NULLIFY_PROOF_ACCOUNTS_LEN { + if proof_accounts_len != NULLIFY_2_PROOF_ACCOUNTS_LEN { return err!(RegistryError::InvalidProofAccountsLength); } Ok(()) @@ -117,18 +122,18 @@ pub(crate) fn validate_compact_nullify_inputs( #[cfg(test)] mod tests { - use super::validate_compact_nullify_inputs; + use super::validate_nullify_2_inputs; use crate::errors::RegistryError; #[test] - fn compact_nullify_inputs_validate_happy_path() { - let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 16); + fn nullify_2_inputs_validate_happy_path() { + let result = validate_nullify_2_inputs(&[1], &[1], &[42], 16); assert!(result.is_ok()); } #[test] - fn compact_nullify_inputs_reject_empty_proof_accounts() { - let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 0); + fn nullify_2_inputs_reject_empty_proof_accounts() { + let result = validate_nullify_2_inputs(&[1], &[1], &[42], 0); assert_eq!( result.err().unwrap(), RegistryError::EmptyProofAccounts.into() @@ -136,8 +141,8 @@ mod tests { } #[test] - fn compact_nullify_inputs_reject_vector_length_mismatch() { - let result = validate_compact_nullify_inputs(&[1, 2], &[1], &[42], 16); + fn nullify_2_inputs_reject_vector_length_mismatch() { + let result = validate_nullify_2_inputs(&[1, 2], &[1], &[42], 16); assert_eq!( result.err().unwrap(), RegistryError::InvalidCompactNullifyInputs.into() @@ -145,8 +150,8 @@ mod tests { } #[test] - fn compact_nullify_inputs_reject_invalid_proof_accounts_length() { - let result = validate_compact_nullify_inputs(&[1], &[1], &[42], 15); + fn nullify_2_inputs_reject_invalid_proof_accounts_length() { + let result = validate_nullify_2_inputs(&[1], &[1], &[42], 15); assert_eq!( result.err().unwrap(), RegistryError::InvalidProofAccountsLength.into() diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 3ef001ce77..42d81c43fd 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -62,7 +62,7 @@ pub fn create_nullify_instruction( } } -pub fn create_nullify_with_proof_accounts_instruction( +pub fn create_nullify_2_instruction( inputs: CreateNullifyInstructionInputs, epoch: u64, ) -> Instruction { @@ -73,7 +73,7 @@ pub fn create_nullify_with_proof_accounts_instruction( Some(get_forester_epoch_pda_from_authority(&inputs.derivation, epoch).0) }; let (cpi_authority, bump) = get_cpi_authority_pda(); - let instruction_data = crate::instruction::NullifyWithProofAccounts { + let instruction_data = crate::instruction::Nullify2 { bump, change_log_indices: inputs.change_log_indices, leaves_queue_indices: inputs.leaves_queue_indices, @@ -632,7 +632,7 @@ mod tests { } #[test] - fn create_nullify_with_proof_accounts_instruction_uses_compact_payload_and_remaining_accounts() + fn create_nullify_2_instruction_uses_compact_payload_and_remaining_accounts() { let authority = Pubkey::new_unique(); let derivation = Pubkey::new_unique(); @@ -645,7 +645,7 @@ mod tests { node }) .collect::>(); - let ix = create_nullify_with_proof_accounts_instruction( + let ix = create_nullify_2_instruction( CreateNullifyInstructionInputs { authority, nullifier_queue, @@ -670,7 +670,7 @@ mod tests { assert_eq!( &ix.data[..8], - crate::instruction::NullifyWithProofAccounts::DISCRIMINATOR + crate::instruction::Nullify2::DISCRIMINATOR ); // 8-byte discriminator + 31-byte compact payload. assert_eq!(ix.data.len(), 39); diff --git a/programs/registry/src/lib.rs b/programs/registry/src/lib.rs index 3caa70e025..e2656f0ca9 100644 --- a/programs/registry/src/lib.rs +++ b/programs/registry/src/lib.rs @@ -420,7 +420,7 @@ pub mod light_registry { ) } - pub fn nullify_with_proof_accounts<'info>( + pub fn nullify_2<'info>( ctx: Context<'_, '_, '_, 'info, NullifyLeaves<'info>>, bump: u8, change_log_indices: Vec, @@ -436,7 +436,7 @@ pub mod light_registry { DEFAULT_WORK_V1, )?; - process_nullify_from_remaining_accounts( + process_nullify_2( &ctx, bump, change_log_indices, From a7652e6578993f1feaf38cad1279e3bc808c473e Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:17:31 +0000 Subject: [PATCH 06/12] rename to 2 --- forester/docs/v1_forester_flows.md | 13 +++++++------ ...ullify_regression.rs => nullify_2_regression.rs} | 4 ++-- .../registry/src/account_compression_cpi/nullify.rs | 4 ++-- .../registry/src/account_compression_cpi/sdk.rs | 4 ++-- programs/registry/src/errors.rs | 6 +++--- 5 files changed, 16 insertions(+), 15 deletions(-) rename program-tests/registry-test/tests/{compact_nullify_regression.rs => nullify_2_regression.rs} (98%) diff --git a/forester/docs/v1_forester_flows.md b/forester/docs/v1_forester_flows.md index 6f6b1d1d53..1a064e8b23 100644 --- a/forester/docs/v1_forester_flows.md +++ b/forester/docs/v1_forester_flows.md @@ -1,4 +1,4 @@ -# Forester V1 Flows (PR: Compact Nullify + Blockhash) +# Forester V1 Flows (PR: v2 Nullify + Blockhash) ## 1. Transaction Send Flow (Blockhash) @@ -54,14 +54,14 @@ No refetch-before-send. No re-sign. ``` -## 2. State Nullify Instruction Flow (Compact vs Legacy) +## 2. State Nullify Instruction Flow (Legacy vs v2) ``` ┌─────────────────────────────────────────────────────────────────────────────────┐ │ Registry: nullify instruction paths │ └─────────────────────────────────────────────────────────────────────────────────┘ - LEGACY (proof in ix data) COMPACT (proof in remaining_accounts) + LEGACY (proof in ix data) v2 (proof in remaining_accounts) ─────────────────────── ──────────────────────────────────── create_nullify_instruction() create_nullify_with_proof_accounts_instruction() @@ -79,7 +79,7 @@ - extract_proof_nodes_from_remaining_accounts - process_nullify(..., vec![proof_nodes]) - Forester V1 uses COMPACT only (create_nullify_with_proof_accounts_instruction). + Forester V1 uses V2 only (create_nullify_with_proof_accounts_instruction). ``` ## 3. Forester V1 State Nullify Pairing Flow @@ -92,7 +92,7 @@ fetch_proofs_and_create_instructions │ │ For each state item: - │ create_nullify_with_proof_accounts_instruction (compact) + │ create_nullify_with_proof_accounts_instruction (v2) │ → StateNullifyInstruction { instruction, proof_nodes, leaf_index } │ ▼ @@ -147,7 +147,7 @@ build_signed_transaction_batch │ ├─ fetch_proofs_and_create_instructions - │ - state: compact nullify ix (proof in remaining_accounts) + │ - state: v2 nullify ix (proof in remaining_accounts) │ - address: update ix │ ├─ build_instruction_batches @@ -161,3 +161,4 @@ - PreparedTransaction::legacy(tx, chunk_last_valid_block_height) - send + confirm with blockhash expiry check ``` + diff --git a/program-tests/registry-test/tests/compact_nullify_regression.rs b/program-tests/registry-test/tests/nullify_2_regression.rs similarity index 98% rename from program-tests/registry-test/tests/compact_nullify_regression.rs rename to program-tests/registry-test/tests/nullify_2_regression.rs index 93f7b83ff1..6b21f04941 100644 --- a/program-tests/registry-test/tests/compact_nullify_regression.rs +++ b/program-tests/registry-test/tests/nullify_2_regression.rs @@ -24,7 +24,7 @@ use solana_sdk::{ #[serial] #[tokio::test] -async fn test_compact_nullify_validation_and_success() { +async fn test_nullify_2_validation_and_success() { let mut rpc = LightProgramTest::new(ProgramTestConfig::default_with_batched_trees(true)) .await .unwrap(); @@ -139,7 +139,7 @@ async fn test_compact_nullify_validation_and_success() { let result = rpc .create_and_send_transaction(&[malformed_ix], &forester.pubkey(), &[&forester]) .await; - assert_rpc_error(result, 0, RegistryError::InvalidCompactNullifyInputs.into()).unwrap(); + assert_rpc_error(result, 0, RegistryError::InvalidNullify2Inputs.into()).unwrap(); rpc.create_and_send_transaction(&[valid_ix], &forester.pubkey(), &[&forester]) .await diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index 0bad92ad45..eae3bafb94 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -109,7 +109,7 @@ pub(crate) fn validate_nullify_2_inputs( || leaves_queue_indices.len() != 1 || indices.len() != 1 { - return err!(RegistryError::InvalidCompactNullifyInputs); + return err!(RegistryError::InvalidNullify2Inputs); } if proof_accounts_len == 0 { return err!(RegistryError::EmptyProofAccounts); @@ -145,7 +145,7 @@ mod tests { let result = validate_nullify_2_inputs(&[1, 2], &[1], &[42], 16); assert_eq!( result.err().unwrap(), - RegistryError::InvalidCompactNullifyInputs.into() + RegistryError::InvalidNullify2Inputs.into() ); } diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 42d81c43fd..01090deff0 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -632,7 +632,7 @@ mod tests { } #[test] - fn create_nullify_2_instruction_uses_compact_payload_and_remaining_accounts() + fn create_nullify_2_instruction_uses_minimal_payload_and_remaining_accounts() { let authority = Pubkey::new_unique(); let derivation = Pubkey::new_unique(); @@ -672,7 +672,7 @@ mod tests { &ix.data[..8], crate::instruction::Nullify2::DISCRIMINATOR ); - // 8-byte discriminator + 31-byte compact payload. + // 8-byte discriminator + 31-byte minimal payload. assert_eq!(ix.data.len(), 39); } } diff --git a/programs/registry/src/errors.rs b/programs/registry/src/errors.rs index 41cfb9987c..a47293e823 100644 --- a/programs/registry/src/errors.rs +++ b/programs/registry/src/errors.rs @@ -40,10 +40,10 @@ pub enum RegistryError { EmptyIndices, #[msg("Proof accounts cannot be empty")] EmptyProofAccounts, - #[msg("Compact nullify proof accounts length is invalid")] + #[msg("Nullify2 proof accounts length is invalid")] InvalidProofAccountsLength, - #[msg("Compact nullify supports exactly one change, queue index, and leaf index")] - InvalidCompactNullifyInputs, + #[msg("Nullify2 supports exactly one change, queue index, and leaf index")] + InvalidNullify2Inputs, #[msg("Failed to borrow account data")] BorrowAccountDataFailed, #[msg("Failed to serialize instruction data")] From 0a0ed7900121e169c908d7df227d811f659759ab Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:21:11 +0000 Subject: [PATCH 07/12] simpler ixn --- forester/docs/v1_forester_flows.md | 4 +-- .../src/account_compression_cpi/nullify.rs | 28 +------------------ programs/registry/src/lib.rs | 15 +++++++++- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/forester/docs/v1_forester_flows.md b/forester/docs/v1_forester_flows.md index 1a064e8b23..8f1c77489f 100644 --- a/forester/docs/v1_forester_flows.md +++ b/forester/docs/v1_forester_flows.md @@ -73,13 +73,13 @@ │ (authority, merkle_tree, queue...) │ account pubkeys (key = node bytes) │ │ ▼ ▼ - process_nullify() process_nullify_2() + process_nullify() nullify_2 instruction (proofs from ix data) - validate: 1 change, 1 queue, 1 index - validate: exactly 16 proof accounts - extract_proof_nodes_from_remaining_accounts - process_nullify(..., vec![proof_nodes]) - Forester V1 uses V2 only (create_nullify_with_proof_accounts_instruction). + Forester V1 uses nullify_2 only (create_nullify_2_instruction). ``` ## 3. Forester V1 State Nullify Pairing Flow diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index eae3bafb94..d5195f5799 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -64,33 +64,7 @@ pub fn process_nullify( ) } -pub fn process_nullify_2( - ctx: &Context, - bump: u8, - change_log_indices: Vec, - leaves_queue_indices: Vec, - indices: Vec, -) -> Result<()> { - validate_nullify_2_inputs( - &change_log_indices, - &leaves_queue_indices, - &indices, - ctx.remaining_accounts.len(), - )?; - - let proof_nodes = extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); - - process_nullify( - ctx, - bump, - change_log_indices, - leaves_queue_indices, - indices, - vec![proof_nodes], - ) -} - -fn extract_proof_nodes_from_remaining_accounts( +pub(crate) fn extract_proof_nodes_from_remaining_accounts( remaining_accounts: &[AccountInfo<'_>], ) -> Vec<[u8; 32]> { remaining_accounts diff --git a/programs/registry/src/lib.rs b/programs/registry/src/lib.rs index e2656f0ca9..7ee5fa25c2 100644 --- a/programs/registry/src/lib.rs +++ b/programs/registry/src/lib.rs @@ -18,6 +18,9 @@ pub use account_compression_cpi::{ rollover_batched_address_tree::*, rollover_batched_state_tree::*, rollover_state_tree::*, update_address_tree::*, }; +use account_compression_cpi::nullify::{ + extract_proof_nodes_from_remaining_accounts, validate_nullify_2_inputs, +}; pub use compressible::{ claim::*, compress_and_close::*, create_config::*, create_config_counter::*, update_config::*, withdraw_funding_pool::*, @@ -436,12 +439,22 @@ pub mod light_registry { DEFAULT_WORK_V1, )?; - process_nullify_2( + validate_nullify_2_inputs( + &change_log_indices, + &leaves_queue_indices, + &indices, + ctx.remaining_accounts.len(), + )?; + let proof_nodes = + extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); + + process_nullify( &ctx, bump, change_log_indices, leaves_queue_indices, indices, + vec![proof_nodes], ) } From d34669f674156d748817895a3e8662820076d976 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:31:06 +0000 Subject: [PATCH 08/12] stricter client api for nullify_2 --- forester/src/processor/v1/helpers.rs | 14 ++++--- .../tests/nullify_2_regression.rs | 13 +++--- .../src/account_compression_cpi/sdk.rs | 42 ++++++++++++------- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index 6f37fa8128..20b5a8a9ee 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -12,7 +12,7 @@ use light_client::{indexer::Indexer, rpc::Rpc}; use light_compressed_account::TreeType; use light_registry::account_compression_cpi::sdk::{ create_nullify_2_instruction, create_update_address_merkle_tree_instruction, - CreateNullifyInstructionInputs, UpdateAddressMerkleTreeInstructionInputs, + CreateNullify2InstructionInputs, UpdateAddressMerkleTreeInstructionInputs, }; use solana_program::instruction::Instruction; use tokio::time::Instant; @@ -389,13 +389,15 @@ pub async fn fetch_proofs_and_create_instructions( proofs.push(MerkleProofType::StateProof(proof.clone())); let instruction = create_nullify_2_instruction( - CreateNullifyInstructionInputs { + CreateNullify2InstructionInputs { nullifier_queue: item.tree_account.queue, merkle_tree: item.tree_account.merkle_tree, - change_log_indices: vec![proof.root_seq % STATE_MERKLE_TREE_CHANGELOG], - leaves_queue_indices: vec![item.queue_item_data.index as u16], - indices: vec![proof.leaf_index], - proofs: vec![proof.proof.clone()], + change_log_index: proof.root_seq % STATE_MERKLE_TREE_CHANGELOG, + leaves_queue_index: item.queue_item_data.index as u16, + index: proof.leaf_index, + proof: proof.proof.clone().try_into().map_err(|_| ForesterError::General { + error: "Failed to convert state proof to fixed array".to_string(), + })?, authority, derivation, is_metadata_forester: false, diff --git a/program-tests/registry-test/tests/nullify_2_regression.rs b/program-tests/registry-test/tests/nullify_2_regression.rs index 6b21f04941..696809ccc5 100644 --- a/program-tests/registry-test/tests/nullify_2_regression.rs +++ b/program-tests/registry-test/tests/nullify_2_regression.rs @@ -10,7 +10,8 @@ use light_program_test::{ }; use light_registry::{ account_compression_cpi::sdk::{ - create_nullify_2_instruction, create_nullify_instruction, CreateNullifyInstructionInputs, + create_nullify_2_instruction, create_nullify_instruction, + CreateNullify2InstructionInputs, CreateNullifyInstructionInputs, }, errors::RegistryError, }; @@ -98,14 +99,14 @@ async fn test_nullify_2_validation_and_success() { let change_log_index = onchain_tree.changelog_index() as u64; let valid_ix = create_nullify_2_instruction( - CreateNullifyInstructionInputs { + CreateNullify2InstructionInputs { authority: forester.pubkey(), nullifier_queue: state_tree_bundle.accounts.nullifier_queue, merkle_tree: state_tree_bundle.accounts.merkle_tree, - change_log_indices: vec![change_log_index], - leaves_queue_indices: vec![queue_index], - indices: vec![leaf_index], - proofs: vec![proof], + change_log_index, + leaves_queue_index: queue_index, + index: leaf_index, + proof: proof.try_into().unwrap(), derivation: forester.pubkey(), is_metadata_forester: true, }, diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 01090deff0..1946393f54 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -26,6 +26,18 @@ pub struct CreateNullifyInstructionInputs { pub is_metadata_forester: bool, } +pub struct CreateNullify2InstructionInputs { + pub authority: Pubkey, + pub nullifier_queue: Pubkey, + pub merkle_tree: Pubkey, + pub change_log_index: u64, + pub leaves_queue_index: u16, + pub index: u64, + pub proof: [[u8; 32]; 16], + pub derivation: Pubkey, + pub is_metadata_forester: bool, +} + pub fn create_nullify_instruction( inputs: CreateNullifyInstructionInputs, epoch: u64, @@ -63,7 +75,7 @@ pub fn create_nullify_instruction( } pub fn create_nullify_2_instruction( - inputs: CreateNullifyInstructionInputs, + inputs: CreateNullify2InstructionInputs, epoch: u64, ) -> Instruction { let register_program_pda = get_registered_program_pda(&crate::ID); @@ -75,9 +87,9 @@ pub fn create_nullify_2_instruction( let (cpi_authority, bump) = get_cpi_authority_pda(); let instruction_data = crate::instruction::Nullify2 { bump, - change_log_indices: inputs.change_log_indices, - leaves_queue_indices: inputs.leaves_queue_indices, - indices: inputs.indices, + change_log_indices: vec![inputs.change_log_index], + leaves_queue_indices: vec![inputs.leaves_queue_index], + indices: vec![inputs.index], }; let base_accounts = crate::accounts::NullifyLeaves { @@ -91,13 +103,11 @@ pub fn create_nullify_2_instruction( account_compression_program: account_compression::ID, }; let mut accounts = base_accounts.to_account_metas(Some(true)); - for proof in inputs.proofs { - for node in proof { - accounts.push(AccountMeta::new_readonly( - Pubkey::new_from_array(node), - false, - )); - } + for node in inputs.proof { + accounts.push(AccountMeta::new_readonly( + Pubkey::new_from_array(node), + false, + )); } Instruction { @@ -646,14 +656,14 @@ mod tests { }) .collect::>(); let ix = create_nullify_2_instruction( - CreateNullifyInstructionInputs { + CreateNullify2InstructionInputs { authority, nullifier_queue, merkle_tree, - change_log_indices: vec![7], - leaves_queue_indices: vec![11], - indices: vec![42], - proofs: vec![proof.clone()], + change_log_index: 7, + leaves_queue_index: 11, + index: 42, + proof: proof.clone().try_into().unwrap(), derivation, is_metadata_forester: false, }, From ca5dcbf8aedf6b25b9ec37f09bf0a8c5e4d6ab3e Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:37:02 +0000 Subject: [PATCH 09/12] format --- forester/src/processor/v1/helpers.rs | 22 +++++--- forester/src/processor/v1/tx_builder.rs | 33 +++++++----- .../tests/nullify_2_regression.rs | 54 +++++++++---------- .../src/account_compression_cpi/nullify.rs | 5 +- .../src/account_compression_cpi/sdk.rs | 8 +-- programs/registry/src/lib.rs | 9 ++-- 6 files changed, 67 insertions(+), 64 deletions(-) diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index 20b5a8a9ee..6b76eea8ba 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -395,20 +395,26 @@ pub async fn fetch_proofs_and_create_instructions( change_log_index: proof.root_seq % STATE_MERKLE_TREE_CHANGELOG, leaves_queue_index: item.queue_item_data.index as u16, index: proof.leaf_index, - proof: proof.proof.clone().try_into().map_err(|_| ForesterError::General { - error: "Failed to convert state proof to fixed array".to_string(), - })?, + proof: proof + .proof + .clone() + .try_into() + .map_err(|_| ForesterError::General { + error: "Failed to convert state proof to fixed array".to_string(), + })?, authority, derivation, is_metadata_forester: false, }, epoch, ); - instructions.push(PreparedV1Instruction::StateNullify(StateNullifyInstruction { - instruction, - proof_nodes: proof.proof, - leaf_index: proof.leaf_index, - })); + instructions.push(PreparedV1Instruction::StateNullify( + StateNullifyInstruction { + instruction, + proof_nodes: proof.proof, + leaf_index: proof.leaf_index, + }, + )); } Ok((proofs, instructions)) diff --git a/forester/src/processor/v1/tx_builder.rs b/forester/src/processor/v1/tx_builder.rs index a269d1d557..6fe68e517e 100644 --- a/forester/src/processor/v1/tx_builder.rs +++ b/forester/src/processor/v1/tx_builder.rs @@ -2,8 +2,10 @@ use std::{sync::Arc, time::Duration}; use account_compression::processor::initialize_address_merkle_tree::Pubkey; use async_trait::async_trait; +use bincode::serialized_size; use forester_utils::rpc_pool::SolanaRpcPool; use light_client::rpc::Rpc; +use mwmatching::{Matching, SENTINEL}; use solana_program::hash::Hash; use solana_sdk::{ signature::{Keypair, Signer}, @@ -19,15 +21,14 @@ use crate::{ v1::{ config::BuildTransactionBatchConfig, helpers::{ - fetch_proofs_and_create_instructions, PreparedV1Instruction, StateNullifyInstruction, + fetch_proofs_and_create_instructions, PreparedV1Instruction, + StateNullifyInstruction, }, }, }, smart_transaction::{create_smart_transaction, CreateSmartTransactionConfig}, Result, }; -use bincode::serialized_size; -use mwmatching::{Matching, SENTINEL}; const MAX_PAIRING_INSTRUCTIONS: usize = 96; const MAX_PAIR_CANDIDATES: usize = 2_000; @@ -361,13 +362,13 @@ async fn pair_state_nullify_batches( if mate != SENTINEL && mate > i && mate < n { used[i] = true; used[mate] = true; - let (left, right) = - if state_nullify_instructions[i].leaf_index <= state_nullify_instructions[mate].leaf_index - { - (i, mate) - } else { - (mate, i) - }; + let (left, right) = if state_nullify_instructions[i].leaf_index + <= state_nullify_instructions[mate].leaf_index + { + (i, mate) + } else { + (mate, i) + }; let min_leaf = state_nullify_instructions[left].leaf_index; paired_batches.push(( min_leaf, @@ -502,8 +503,14 @@ mod tests { #[test] fn remaining_blocks_guard_is_strictly_greater_than_threshold() { - assert!(!remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING - 1)); - assert!(!remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING)); - assert!(remaining_blocks_allows_pairing(MIN_REMAINING_BLOCKS_FOR_PAIRING + 1)); + assert!(!remaining_blocks_allows_pairing( + MIN_REMAINING_BLOCKS_FOR_PAIRING - 1 + )); + assert!(!remaining_blocks_allows_pairing( + MIN_REMAINING_BLOCKS_FOR_PAIRING + )); + assert!(remaining_blocks_allows_pairing( + MIN_REMAINING_BLOCKS_FOR_PAIRING + 1 + )); } } diff --git a/program-tests/registry-test/tests/nullify_2_regression.rs b/program-tests/registry-test/tests/nullify_2_regression.rs index 696809ccc5..ef6fdfbb5f 100644 --- a/program-tests/registry-test/tests/nullify_2_regression.rs +++ b/program-tests/registry-test/tests/nullify_2_regression.rs @@ -4,14 +4,12 @@ use forester_utils::account_zero_copy::{get_concurrent_merkle_tree, get_hash_set use light_compressed_account::TreeType; use light_hasher::Poseidon; use light_program_test::{ - program_test::LightProgramTest, - utils::assert::assert_rpc_error, - ProgramTestConfig, + program_test::LightProgramTest, utils::assert::assert_rpc_error, ProgramTestConfig, }; use light_registry::{ account_compression_cpi::sdk::{ - create_nullify_2_instruction, create_nullify_instruction, - CreateNullify2InstructionInputs, CreateNullifyInstructionInputs, + create_nullify_2_instruction, create_nullify_instruction, CreateNullify2InstructionInputs, + CreateNullifyInstructionInputs, }, errors::RegistryError, }; @@ -64,9 +62,10 @@ async fn test_nullify_2_validation_and_success() { (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) }; - let nullifier_queue = - unsafe { get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await } - .unwrap(); + let nullifier_queue = unsafe { + get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await + } + .unwrap(); let mut queue_index = None; let mut account_hash = None; for i in 0..nullifier_queue.get_capacity() { @@ -90,12 +89,13 @@ async fn test_nullify_2_validation_and_success() { .get_proof_of_leaf(leaf_index as usize, false) .unwrap(); let proof_depth = proof.len(); - let onchain_tree = get_concurrent_merkle_tree::( - &mut rpc, - state_tree_bundle.accounts.merkle_tree, - ) - .await - .unwrap(); + let onchain_tree = + get_concurrent_merkle_tree::( + &mut rpc, + state_tree_bundle.accounts.merkle_tree, + ) + .await + .unwrap(); let change_log_index = onchain_tree.changelog_index() as u64; let valid_ix = create_nullify_2_instruction( @@ -118,11 +118,7 @@ async fn test_nullify_2_validation_and_success() { .accounts .truncate(empty_proof_accounts_ix.accounts.len() - proof_depth); let result = rpc - .create_and_send_transaction( - &[empty_proof_accounts_ix], - &forester.pubkey(), - &[&forester], - ) + .create_and_send_transaction(&[empty_proof_accounts_ix], &forester.pubkey(), &[&forester]) .await; assert_rpc_error(result, 0, RegistryError::EmptyProofAccounts.into()).unwrap(); @@ -187,9 +183,10 @@ async fn test_legacy_nullify_still_succeeds() { .unwrap(); (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) }; - let nullifier_queue = - unsafe { get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await } - .unwrap(); + let nullifier_queue = unsafe { + get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await + } + .unwrap(); let mut queue_index = None; let mut account_hash = None; for i in 0..nullifier_queue.get_capacity() { @@ -212,12 +209,13 @@ async fn test_legacy_nullify_still_succeeds() { .merkle_tree .get_proof_of_leaf(leaf_index as usize, false) .unwrap(); - let onchain_tree = get_concurrent_merkle_tree::( - &mut rpc, - state_tree_bundle.accounts.merkle_tree, - ) - .await - .unwrap(); + let onchain_tree = + get_concurrent_merkle_tree::( + &mut rpc, + state_tree_bundle.accounts.merkle_tree, + ) + .await + .unwrap(); let change_log_index = onchain_tree.changelog_index() as u64; let legacy_ix = create_nullify_instruction( diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index d5195f5799..5592b21cb0 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -79,10 +79,7 @@ pub(crate) fn validate_nullify_2_inputs( indices: &[u64], proof_accounts_len: usize, ) -> Result<()> { - if change_log_indices.len() != 1 - || leaves_queue_indices.len() != 1 - || indices.len() != 1 - { + if change_log_indices.len() != 1 || leaves_queue_indices.len() != 1 || indices.len() != 1 { return err!(RegistryError::InvalidNullify2Inputs); } if proof_accounts_len == 0 { diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 1946393f54..0e32f17282 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -642,8 +642,7 @@ mod tests { } #[test] - fn create_nullify_2_instruction_uses_minimal_payload_and_remaining_accounts() - { + fn create_nullify_2_instruction_uses_minimal_payload_and_remaining_accounts() { let authority = Pubkey::new_unique(); let derivation = Pubkey::new_unique(); let nullifier_queue = Pubkey::new_unique(); @@ -678,10 +677,7 @@ mod tests { assert!(!account_meta.is_writable); } - assert_eq!( - &ix.data[..8], - crate::instruction::Nullify2::DISCRIMINATOR - ); + assert_eq!(&ix.data[..8], crate::instruction::Nullify2::DISCRIMINATOR); // 8-byte discriminator + 31-byte minimal payload. assert_eq!(ix.data.len(), 39); } diff --git a/programs/registry/src/lib.rs b/programs/registry/src/lib.rs index 7ee5fa25c2..16a5d56548 100644 --- a/programs/registry/src/lib.rs +++ b/programs/registry/src/lib.rs @@ -11,6 +11,9 @@ use light_merkle_tree_metadata::merkle_tree::MerkleTreeMetadata; pub mod account_compression_cpi; pub mod errors; +use account_compression_cpi::nullify::{ + extract_proof_nodes_from_remaining_accounts, validate_nullify_2_inputs, +}; pub use account_compression_cpi::{ batch_append::*, batch_nullify::*, batch_update_address_tree::*, initialize_batched_address_tree::*, initialize_batched_state_tree::*, @@ -18,9 +21,6 @@ pub use account_compression_cpi::{ rollover_batched_address_tree::*, rollover_batched_state_tree::*, rollover_state_tree::*, update_address_tree::*, }; -use account_compression_cpi::nullify::{ - extract_proof_nodes_from_remaining_accounts, validate_nullify_2_inputs, -}; pub use compressible::{ claim::*, compress_and_close::*, create_config::*, create_config_counter::*, update_config::*, withdraw_funding_pool::*, @@ -445,8 +445,7 @@ pub mod light_registry { &indices, ctx.remaining_accounts.len(), )?; - let proof_nodes = - extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); + let proof_nodes = extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); process_nullify( &ctx, From a3c2207c64839b591280a500e3f084e1fd31fb35 Mon Sep 17 00:00:00 2001 From: Swenschaeferjohann Date: Mon, 16 Mar 2026 15:42:13 +0000 Subject: [PATCH 10/12] reduce sig pressure on pairing simulation --- forester/src/processor/v1/tx_builder.rs | 36 ++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/forester/src/processor/v1/tx_builder.rs b/forester/src/processor/v1/tx_builder.rs index 6fe68e517e..76b3fecf60 100644 --- a/forester/src/processor/v1/tx_builder.rs +++ b/forester/src/processor/v1/tx_builder.rs @@ -8,6 +8,7 @@ use light_client::rpc::Rpc; use mwmatching::{Matching, SENTINEL}; use solana_program::hash::Hash; use solana_sdk::{ + compute_budget::ComputeBudgetInstruction, signature::{Keypair, Signer}, transaction::Transaction, }; @@ -323,12 +324,9 @@ async fn pair_state_nullify_batches( &state_nullify_instructions[j].instruction, payer, recent_blockhash, - last_valid_block_height, priority_fee, compute_unit_limit, - ) - .await? - { + ) { continue; } let overlap = state_nullify_instructions[i] @@ -413,24 +411,32 @@ fn remaining_blocks_allows_pairing(remaining_blocks: u64) -> bool { remaining_blocks > MIN_REMAINING_BLOCKS_FOR_PAIRING } -async fn pair_fits_transaction_size( +fn pair_fits_transaction_size( ix_a: &solana_program::instruction::Instruction, ix_b: &solana_program::instruction::Instruction, payer: &Keypair, recent_blockhash: &Hash, - last_valid_block_height: u64, priority_fee: Option, compute_unit_limit: Option, ) -> Result { - let (tx, _) = create_smart_transaction(CreateSmartTransactionConfig { - payer: payer.insecure_clone(), - instructions: vec![ix_a.clone(), ix_b.clone()], - recent_blockhash: *recent_blockhash, - compute_unit_price: priority_fee, - compute_unit_limit, - last_valid_block_height, - }) - .await?; + let mut instructions = Vec::with_capacity( + 2 + usize::from(priority_fee.is_some()) + usize::from(compute_unit_limit.is_some()), + ); + if let Some(price) = priority_fee { + instructions.push(ComputeBudgetInstruction::set_compute_unit_price(price)); + } + if let Some(limit) = compute_unit_limit { + instructions.push(ComputeBudgetInstruction::set_compute_unit_limit(limit)); + } + instructions.push(ix_a.clone()); + instructions.push(ix_b.clone()); + + let mut tx = Transaction::new_with_payer(&instructions, Some(&payer.pubkey())); + tx.message.recent_blockhash = *recent_blockhash; + tx.signatures = vec![ + solana_sdk::signature::Signature::default(); + tx.message.header.num_required_signatures as usize + ]; let tx_bytes = serialized_size(&tx)? as usize; Ok(tx_bytes <= 1232) From 8f7b8210e028d718c906a6a7d8dcbb2127c54009 Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Mon, 16 Mar 2026 17:59:17 +0000 Subject: [PATCH 11/12] improve nullify pairing: replace deps, fix size check, add tests - Replace mwmatching crate with internal Blossom algorithm (matching.rs) - Remove bincode dep; compute tx size natively via wire-format calculation - Align pair_fits_transaction_size with create_smart_transaction path - Add 32-byte safety margin to transaction size check - Use HashSet for O(1) proof-node overlap instead of linear scan - Eliminate instruction clones via Option::take() - Sort state nullifies by leaf_index before pairing for better overlap - Add pairs_only mode to BuildTransactionBatchConfig for fullness-based foresting - Add merkle_tree field to StateNullifyInstruction for future multi-tree grouping - Remove redundant EmptyProofAccounts error (subsumed by InvalidProofAccountsLength) - Fix stale "30 seconds" comment (actual timeout is 15s) - Add unit tests for pair_state_nullify_batches with realistic fixtures --- Cargo.lock | 8 - forester/Cargo.toml | 2 - forester/src/epoch_manager.rs | 1 + forester/src/lib.rs | 1 + forester/src/matching.rs | 325 ++++++++++++ forester/src/processor/v1/config.rs | 4 + forester/src/processor/v1/helpers.rs | 2 + forester/src/processor/v1/tx_builder.rs | 463 +++++++++++++++--- forester/src/smart_transaction.rs | 2 +- .../tests/nullify_2_regression.rs | 2 +- .../src/account_compression_cpi/nullify.rs | 5 +- programs/registry/src/errors.rs | 2 - 12 files changed, 725 insertions(+), 92 deletions(-) create mode 100644 forester/src/matching.rs diff --git a/Cargo.lock b/Cargo.lock index b5710c3afe..9e293fd2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2439,7 +2439,6 @@ dependencies = [ "async-channel 2.5.0", "async-trait", "base64 0.13.1", - "bincode", "borsh 0.10.4", "bs58", "clap 4.5.60", @@ -2473,7 +2472,6 @@ dependencies = [ "light-token", "light-token-client", "light-token-interface", - "mwmatching", "photon-api", "prometheus", "rand 0.8.5", @@ -4838,12 +4836,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "mwmatching" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b50448d988736cc2c938a76ae336241fcb31a225017c0e3121bd349e7dc06" - [[package]] name = "native-tls" version = "0.2.18" diff --git a/forester/Cargo.toml b/forester/Cargo.toml index 8443848391..32d1df4e0d 100644 --- a/forester/Cargo.toml +++ b/forester/Cargo.toml @@ -43,7 +43,6 @@ reqwest = { workspace = true, features = ["json", "rustls-tls", "blocking"] } futures = { workspace = true } thiserror = { workspace = true } borsh = { workspace = true } -bincode = "1.3" bs58 = { workspace = true } hex = { workspace = true } env_logger = { workspace = true } @@ -62,7 +61,6 @@ itertools = "0.14" async-channel = "2.5" solana-pubkey = { workspace = true } dotenvy = "0.15" -mwmatching = "0.1.1" [dev-dependencies] serial_test = { workspace = true } diff --git a/forester/src/epoch_manager.rs b/forester/src/epoch_manager.rs index f52efa1b13..1f4e83dfc5 100644 --- a/forester/src/epoch_manager.rs +++ b/forester/src/epoch_manager.rs @@ -2993,6 +2993,7 @@ impl EpochManager { compute_unit_limit: Some(self.config.transaction_config.cu_limit), enable_priority_fees: self.config.transaction_config.enable_priority_fees, max_concurrent_sends: Some(self.config.transaction_config.max_concurrent_sends), + pairs_only: false, }, queue_config: self.config.queue_config, retry_config: RetryConfig { diff --git a/forester/src/lib.rs b/forester/src/lib.rs index aebb2d4e9f..546e43cf7e 100644 --- a/forester/src/lib.rs +++ b/forester/src/lib.rs @@ -5,6 +5,7 @@ pub mod cli; pub mod compressible; pub mod config; pub mod epoch_manager; +pub(crate) mod matching; pub mod errors; pub mod forester_status; pub mod health_check; diff --git a/forester/src/matching.rs b/forester/src/matching.rs new file mode 100644 index 0000000000..0bdc6ef3db --- /dev/null +++ b/forester/src/matching.rs @@ -0,0 +1,325 @@ +//! Maximum cardinality matching via Edmonds' Blossom algorithm. +//! +//! Finds the largest set of non-overlapping vertex pairs in a general +//! (non-bipartite) weighted graph. When multiple maximum-cardinality +//! matchings exist, the one with the highest total weight is preferred +//! thanks to the greedy initialization sorting edges by weight. + +use std::collections::VecDeque; + +/// Returned for unmatched vertices. +pub const SENTINEL: usize = usize::MAX; + +/// Builder for a maximum-cardinality matching on a general weighted graph. +/// +/// ```ignore +/// let mates = Matching::new(edges).max_cardinality().solve(); +/// ``` +pub struct Matching { + edges: Vec<(usize, usize, i32)>, +} + +impl Matching { + pub fn new(edges: Vec<(usize, usize, i32)>) -> Self { + Self { edges } + } + + /// Request maximum-cardinality mode (currently the only mode). + pub fn max_cardinality(self) -> Self { + self + } + + /// Solve and return `mates[v]` for every vertex `v`. + /// `mates[v] == SENTINEL` when `v` is unmatched. + pub fn solve(self) -> Vec { + if self.edges.is_empty() { + return Vec::new(); + } + let n = self + .edges + .iter() + .flat_map(|&(u, v, _)| [u, v]) + .max() + .map(|m| m + 1) + .unwrap_or(0); + if n == 0 { + return Vec::new(); + } + edmonds_matching(n, &self.edges) + } +} + +// --------------------------------------------------------------------------- +// Edmonds' Blossom algorithm – O(V²·E) maximum-cardinality matching +// --------------------------------------------------------------------------- + +fn edmonds_matching(n: usize, edges: &[(usize, usize, i32)]) -> Vec { + let mut adj = vec![vec![]; n]; + for &(u, v, _) in edges { + adj[u].push(v); + adj[v].push(u); + } + + let mut mate = vec![SENTINEL; n]; + + // Greedy init: prefer higher-weight edges for a better starting point. + let mut sorted: Vec<_> = edges.to_vec(); + sorted.sort_by(|a, b| b.2.cmp(&a.2)); + for &(u, v, _) in &sorted { + if mate[u] == SENTINEL && mate[v] == SENTINEL { + mate[u] = v; + mate[v] = u; + } + } + + // Augment from every remaining free vertex. + for root in 0..n { + if mate[root] != SENTINEL { + continue; + } + try_augment(n, &adj, &mut mate, root); + } + + mate +} + +/// BFS from `root` looking for an augmenting path. Returns `true` if one was +/// found (and the matching has already been updated). +fn try_augment(n: usize, adj: &[Vec], mate: &mut [usize], root: usize) -> bool { + let mut base: Vec = (0..n).collect(); + let mut parent = vec![SENTINEL; n]; + let mut color = vec![0u8; n]; // 0 = unseen, 1 = outer, 2 = inner + let mut queue = VecDeque::new(); + + color[root] = 1; + queue.push_back(root); + + while let Some(v) = queue.pop_front() { + for &u in &adj[v] { + if base[v] == base[u] || color[u] == 2 { + continue; + } + if color[u] == 1 { + // Both outer → blossom. + let lca = find_lca(&base, &parent, mate, root, v, u); + contract( + &mut base, + &mut parent, + &mut color, + mate, + &mut queue, + v, + u, + lca, + ); + } else if mate[u] == SENTINEL { + // Free vertex → augmenting path found. + parent[u] = v; + augment(mate, &parent, u); + return true; + } else { + // Matched, unseen vertex → extend tree. + parent[u] = v; + color[u] = 2; + let w = mate[u]; + color[w] = 1; + queue.push_back(w); + } + } + } + + false +} + +/// Walk from both endpoints towards the root to find the lowest common +/// ancestor in the alternating tree (respecting blossom bases). +fn find_lca( + base: &[usize], + parent: &[usize], + mate: &[usize], + root: usize, + a: usize, + b: usize, +) -> usize { + let n = base.len(); + let mut visited = vec![false; n]; + let mut a = base[a]; + let mut b = base[b]; + loop { + visited[a] = true; + if a == root { + break; + } + a = base[parent[mate[a]]]; + } + loop { + if visited[b] { + return b; + } + b = base[parent[mate[b]]]; + } +} + +/// Shrink the blossom defined by paths v→lca and u→lca. +fn contract( + base: &mut [usize], + parent: &mut [usize], + color: &mut [u8], + mate: &[usize], + queue: &mut VecDeque, + v: usize, + u: usize, + lca: usize, +) { + let n = base.len(); + let mut blossom = vec![false; n]; + mark_path(base, parent, mate, &mut blossom, v, lca, u); + mark_path(base, parent, mate, &mut blossom, u, lca, v); + for i in 0..n { + if blossom[base[i]] { + base[i] = lca; + if color[i] != 1 { + color[i] = 1; + queue.push_back(i); + } + } + } +} + +/// Walk from `v` towards `lca`, marking blossom members and redirecting +/// parent pointers so that future augmentations can traverse the blossom. +fn mark_path( + base: &[usize], + parent: &mut [usize], + mate: &[usize], + blossom: &mut [bool], + mut v: usize, + lca: usize, + child: usize, +) { + let mut cur_child = child; + while base[v] != lca { + blossom[base[v]] = true; + blossom[base[mate[v]]] = true; + parent[v] = cur_child; + cur_child = mate[v]; + v = parent[mate[v]]; + } +} + +/// Flip matched / unmatched edges along the augmenting path ending at `u`. +fn augment(mate: &mut [usize], parent: &[usize], mut u: usize) { + while u != SENTINEL { + let v = parent[u]; + let prev = if v != SENTINEL { mate[v] } else { SENTINEL }; + mate[u] = v; + if v != SENTINEL { + mate[v] = u; + } + u = prev; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prioritizes_cardinality() { + // Path 0–1–2: one edge can be matched. With max_cardinality the + // algorithm must NOT leave node 1 unmatched just because (0,1) has + // higher weight. Either (0,1) or (1,2) is fine – 1 pair total. + let edges = vec![(0, 1, 10_100i32), (1, 2, 10_090)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs: Vec<_> = mates + .iter() + .enumerate() + .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .collect(); + assert_eq!(pairs.len(), 1); + } + + #[test] + fn disconnected_components() { + let edges = vec![(0, 1, 10_010), (2, 3, 10_005)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let matched = mates.iter().filter(|&&m| m != SENTINEL).count(); + assert_eq!(matched, 4); + } + + #[test] + fn empty_edges() { + let mates = Matching::new(vec![]).max_cardinality().solve(); + assert!(mates.is_empty()); + } + + #[test] + fn triangle() { + // Triangle: max matching = 1 pair. + let edges = vec![(0, 1, 1), (1, 2, 1), (0, 2, 1)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let matched = mates.iter().filter(|&&m| m != SENTINEL).count(); + assert_eq!(matched, 2); // 1 pair = 2 matched vertices + } + + #[test] + fn augmenting_path_needed() { + // 0–1–2–3: greedy might match (0,1) and leave (2,3) for the second + // pass. Either way, 2 pairs are achievable. + let edges = vec![(0, 1, 1), (1, 2, 1), (2, 3, 1)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs: Vec<_> = mates + .iter() + .enumerate() + .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .collect(); + assert_eq!(pairs.len(), 2); + } + + #[test] + fn pentagon_blossom() { + // 5-cycle: max matching = 2 pairs. + let edges = vec![(0, 1, 1), (1, 2, 1), (2, 3, 1), (3, 4, 1), (4, 0, 1)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs: Vec<_> = mates + .iter() + .enumerate() + .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .collect(); + assert_eq!(pairs.len(), 2); + } + + #[test] + fn complete_graph_k6() { + // K6: 6 vertices, max matching = 3 pairs. + let mut edges = Vec::new(); + for i in 0..6 { + for j in (i + 1)..6 { + edges.push((i, j, 1)); + } + } + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs: Vec<_> = mates + .iter() + .enumerate() + .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .collect(); + assert_eq!(pairs.len(), 3); + } + + #[test] + fn weight_tiebreaker() { + // 4 vertices, 2 possible perfect matchings: + // (0,1)+(2,3) total weight = 100+1 = 101 + // (0,2)+(1,3) total weight = 50+50 = 100 + // Greedy init prefers (0,1) first (weight 100), then (2,3). + let edges = vec![(0, 1, 100), (0, 2, 50), (1, 3, 50), (2, 3, 1)]; + let mates = Matching::new(edges).max_cardinality().solve(); + let pairs: Vec<_> = mates + .iter() + .enumerate() + .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .collect(); + assert_eq!(pairs.len(), 2); // perfect matching + } +} diff --git a/forester/src/processor/v1/config.rs b/forester/src/processor/v1/config.rs index f2ee05f353..bba3db3a86 100644 --- a/forester/src/processor/v1/config.rs +++ b/forester/src/processor/v1/config.rs @@ -28,4 +28,8 @@ pub struct BuildTransactionBatchConfig { pub compute_unit_limit: Option, pub enable_priority_fees: bool, pub max_concurrent_sends: Option, + /// When `true`, only emit paired state-nullify transactions. + /// Unpaired (single) nullifies are dropped. The caller sets this based + /// on queue fullness: `pairs_only = queue_pending < MAX_QUEUE_FULLNESS`. + pub pairs_only: bool, } diff --git a/forester/src/processor/v1/helpers.rs b/forester/src/processor/v1/helpers.rs index 6b76eea8ba..e465e260da 100644 --- a/forester/src/processor/v1/helpers.rs +++ b/forester/src/processor/v1/helpers.rs @@ -43,6 +43,7 @@ pub struct StateNullifyInstruction { pub instruction: Instruction, pub proof_nodes: Vec<[u8; 32]>, pub leaf_index: u64, + pub merkle_tree: Pubkey, } /// Work items should be of only one type and tree @@ -413,6 +414,7 @@ pub async fn fetch_proofs_and_create_instructions( instruction, proof_nodes: proof.proof, leaf_index: proof.leaf_index, + merkle_tree: item.tree_account.merkle_tree, }, )); } diff --git a/forester/src/processor/v1/tx_builder.rs b/forester/src/processor/v1/tx_builder.rs index 76b3fecf60..4b5d495860 100644 --- a/forester/src/processor/v1/tx_builder.rs +++ b/forester/src/processor/v1/tx_builder.rs @@ -1,14 +1,11 @@ -use std::{sync::Arc, time::Duration}; +use std::{collections::HashSet, sync::Arc, time::Duration}; use account_compression::processor::initialize_address_merkle_tree::Pubkey; use async_trait::async_trait; -use bincode::serialized_size; use forester_utils::rpc_pool::SolanaRpcPool; use light_client::rpc::Rpc; -use mwmatching::{Matching, SENTINEL}; use solana_program::hash::Hash; use solana_sdk::{ - compute_budget::ComputeBudgetInstruction, signature::{Keypair, Signer}, transaction::Transaction, }; @@ -17,6 +14,7 @@ use tracing::{trace, warn}; use crate::{ epoch_manager::WorkItem, + matching::{Matching, SENTINEL}, processor::{ tx_cache::ProcessedHashCache, v1::{ @@ -27,14 +25,29 @@ use crate::{ }, }, }, - smart_transaction::{create_smart_transaction, CreateSmartTransactionConfig}, + smart_transaction::{ + create_smart_transaction, with_compute_budget_instructions, ComputeBudgetConfig, + CreateSmartTransactionConfig, + }, Result, }; -const MAX_PAIRING_INSTRUCTIONS: usize = 96; -const MAX_PAIR_CANDIDATES: usize = 2_000; +const MAX_PAIRING_INSTRUCTIONS: usize = 100; +const MAX_PAIR_CANDIDATES: usize = 4_950; const MIN_REMAINING_BLOCKS_FOR_PAIRING: u64 = 25; +/// Safety margin subtracted from the Solana packet size (1232 bytes) when +/// checking whether two instructions fit in a single transaction. This +/// accounts for any minor divergence between the size-check path and the +/// real `create_smart_transaction` path (e.g. signature encoding). +const TX_SIZE_SAFETY_MARGIN: usize = 32; + +/// Maximum legacy transaction size (Solana PACKET_DATA_SIZE). +const PACKET_DATA_SIZE: usize = 1232; + +/// Maximum allowed serialised transaction size for a paired batch. +const MAX_TRANSACTION_SIZE: usize = PACKET_DATA_SIZE - TX_SIZE_SAFETY_MARGIN; + #[async_trait] #[allow(clippy::too_many_arguments)] pub trait TransactionBuilder: Send + Sync { @@ -150,7 +163,7 @@ impl TransactionBuilder for EpochManagerTransactions { }) .collect(); - // Add items with short timeout (30 seconds) for processing + // Add items with a short timeout (15 seconds) for processing. for item in &work_items { let hash_str = bs58::encode(&item.queue_item_data.hash).into_string(); cache.add_with_timeout(&hash_str, Duration::from_secs(15)); @@ -216,13 +229,12 @@ impl TransactionBuilder for EpochManagerTransactions { prepared_instructions, batch_size, allow_pairing, + config.pairs_only, payer, recent_blockhash, - last_valid_block_height, priority_fee, config.compute_unit_limit, - ) - .await?; + )?; for instruction_chunk in instruction_batches { let (transaction, _) = create_smart_transaction(CreateSmartTransactionConfig { @@ -252,13 +264,18 @@ impl TransactionBuilder for EpochManagerTransactions { } } -async fn build_instruction_batches( +// --------------------------------------------------------------------------- +// Instruction batching with optional pairing +// --------------------------------------------------------------------------- + +#[allow(clippy::too_many_arguments)] +fn build_instruction_batches( prepared_instructions: Vec, batch_size: usize, allow_pairing: bool, + pairs_only: bool, payer: &Keypair, recent_blockhash: &Hash, - last_valid_block_height: u64, priority_fee: Option, compute_unit_limit: Option, ) -> Result>> { @@ -280,42 +297,59 @@ async fn build_instruction_batches( return Ok(batches); } + // Sort by leaf_index for better proof-node overlap between neighbours. + state_nullify_instructions.sort_by_key(|ix| ix.leaf_index); + let paired_batches = if batch_size >= 2 && allow_pairing { pair_state_nullify_batches( state_nullify_instructions, payer, recent_blockhash, - last_valid_block_height, priority_fee, compute_unit_limit, - ) - .await? - } else { + pairs_only, + )? + } else if !pairs_only { state_nullify_instructions .into_iter() .map(|ix| vec![ix.instruction]) .collect() + } else { + Vec::new() }; batches.extend(paired_batches); Ok(batches) } -async fn pair_state_nullify_batches( +fn pair_state_nullify_batches( state_nullify_instructions: Vec, payer: &Keypair, recent_blockhash: &Hash, - last_valid_block_height: u64, priority_fee: Option, compute_unit_limit: Option, + pairs_only: bool, ) -> Result>> { let n = state_nullify_instructions.len(); if n < 2 { + if pairs_only { + return Ok(Vec::new()); + } return Ok(state_nullify_instructions .into_iter() .map(|ix| vec![ix.instruction]) .collect()); } + // Pre-compute HashSets for O(1) overlap lookup. + let proof_sets: Vec> = state_nullify_instructions + .iter() + .map(|ix| ix.proof_nodes.iter().copied().collect()) + .collect(); + let leaf_indices: Vec = state_nullify_instructions + .iter() + .map(|ix| ix.leaf_index) + .collect(); + let mut edges: Vec<(usize, usize, i32)> = Vec::new(); for i in 0..n { for j in (i + 1)..n { @@ -329,11 +363,7 @@ async fn pair_state_nullify_batches( ) { continue; } - let overlap = state_nullify_instructions[i] - .proof_nodes - .iter() - .filter(|node| state_nullify_instructions[j].proof_nodes.contains(node)) - .count() as i32; + let overlap = proof_sets[i].intersection(&proof_sets[j]).count() as i32; // Prioritize pair count first, then maximize proof overlap. let weight = 10_000 + overlap; edges.push((i, j, weight)); @@ -341,6 +371,9 @@ async fn pair_state_nullify_batches( } if edges.is_empty() { + if pairs_only { + return Ok(Vec::new()); + } return Ok(state_nullify_instructions .into_iter() .map(|ix| vec![ix.instruction]) @@ -348,9 +381,16 @@ async fn pair_state_nullify_batches( } let mates = Matching::new(edges).max_cardinality().solve(); + + // Move instructions into Options for zero-copy extraction. + let mut instructions: Vec> = + state_nullify_instructions + .into_iter() + .map(|ix| Some(ix.instruction)) + .collect(); + let mut used = vec![false; n]; let mut paired_batches: Vec<(u64, Vec)> = Vec::new(); - let mut single_batches: Vec<(u64, Vec)> = Vec::new(); for i in 0..n { if used[i] { @@ -360,30 +400,28 @@ async fn pair_state_nullify_batches( if mate != SENTINEL && mate > i && mate < n { used[i] = true; used[mate] = true; - let (left, right) = if state_nullify_instructions[i].leaf_index - <= state_nullify_instructions[mate].leaf_index - { + let (left, right) = if leaf_indices[i] <= leaf_indices[mate] { (i, mate) } else { (mate, i) }; - let min_leaf = state_nullify_instructions[left].leaf_index; + let min_leaf = leaf_indices[left]; paired_batches.push(( min_leaf, vec![ - state_nullify_instructions[left].instruction.clone(), - state_nullify_instructions[right].instruction.clone(), + instructions[left].take().unwrap(), + instructions[right].take().unwrap(), ], )); } } - for i in 0..n { - if !used[i] { - single_batches.push(( - state_nullify_instructions[i].leaf_index, - vec![state_nullify_instructions[i].instruction.clone()], - )); + let mut single_batches: Vec<(u64, Vec)> = Vec::new(); + if !pairs_only { + for (i, ix) in instructions.into_iter().enumerate() { + if let Some(ix) = ix { + single_batches.push((leaf_indices[i], vec![ix])); + } } } @@ -393,6 +431,77 @@ async fn pair_state_nullify_batches( Ok(paired_batches.into_iter().map(|(_, batch)| batch).collect()) } +// --------------------------------------------------------------------------- +// Transaction-size estimation (no bincode – native wire-format calculation) +// --------------------------------------------------------------------------- + +/// Check whether two instructions plus compute-budget prefixes fit inside a +/// single Solana legacy transaction. Uses the same construction path as +/// [`create_smart_transaction`] to avoid divergence. +fn pair_fits_transaction_size( + ix_a: &solana_program::instruction::Instruction, + ix_b: &solana_program::instruction::Instruction, + payer: &Keypair, + _recent_blockhash: &Hash, + priority_fee: Option, + compute_unit_limit: Option, +) -> bool { + // Build instructions exactly as create_smart_transaction does. + let final_instructions = with_compute_budget_instructions( + vec![ix_a.clone(), ix_b.clone()], + ComputeBudgetConfig { + compute_unit_price: priority_fee, + compute_unit_limit, + }, + ); + let tx = Transaction::new_with_payer(&final_instructions, Some(&payer.pubkey())); + legacy_transaction_size(&tx) <= MAX_TRANSACTION_SIZE +} + +/// Compute the Solana legacy-transaction wire-format size without pulling in +/// a serialisation crate. +fn legacy_transaction_size(tx: &Transaction) -> usize { + let msg = &tx.message; + let num_sigs = msg.header.num_required_signatures as usize; + + // signatures section: compact-u16(count) + count * 64 + let sigs = short_vec_len(num_sigs) + num_sigs * 64; + + // message header (3 bytes) + let header = 3; + + // account keys: compact-u16(count) + count * 32 + let keys = short_vec_len(msg.account_keys.len()) + msg.account_keys.len() * 32; + + // recent_blockhash + let blockhash = 32; + + // instructions: compact-u16(count) + each instruction + let mut ixs = short_vec_len(msg.instructions.len()); + for ix in &msg.instructions { + ixs += 1; // program_id_index (u8) + ixs += short_vec_len(ix.accounts.len()) + ix.accounts.len(); + ixs += short_vec_len(ix.data.len()) + ix.data.len(); + } + + sigs + header + keys + blockhash + ixs +} + +/// Length of a Solana ShortVec (compact-u16) encoding. +fn short_vec_len(val: usize) -> usize { + if val < 0x80 { + 1 + } else if val < 0x4000 { + 2 + } else { + 3 + } +} + +// --------------------------------------------------------------------------- +// Guard helpers +// --------------------------------------------------------------------------- + fn pairing_candidate_count(n: usize) -> usize { n.saturating_sub(1).saturating_mul(n) / 2 } @@ -411,41 +520,19 @@ fn remaining_blocks_allows_pairing(remaining_blocks: u64) -> bool { remaining_blocks > MIN_REMAINING_BLOCKS_FOR_PAIRING } -fn pair_fits_transaction_size( - ix_a: &solana_program::instruction::Instruction, - ix_b: &solana_program::instruction::Instruction, - payer: &Keypair, - recent_blockhash: &Hash, - priority_fee: Option, - compute_unit_limit: Option, -) -> Result { - let mut instructions = Vec::with_capacity( - 2 + usize::from(priority_fee.is_some()) + usize::from(compute_unit_limit.is_some()), - ); - if let Some(price) = priority_fee { - instructions.push(ComputeBudgetInstruction::set_compute_unit_price(price)); - } - if let Some(limit) = compute_unit_limit { - instructions.push(ComputeBudgetInstruction::set_compute_unit_limit(limit)); - } - instructions.push(ix_a.clone()); - instructions.push(ix_b.clone()); - - let mut tx = Transaction::new_with_payer(&instructions, Some(&payer.pubkey())); - tx.message.recent_blockhash = *recent_blockhash; - tx.signatures = vec![ - solana_sdk::signature::Signature::default(); - tx.message.header.num_required_signatures as usize - ]; - - let tx_bytes = serialized_size(&tx)? as usize; - Ok(tx_bytes <= 1232) -} +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- #[cfg(test)] mod tests { + use solana_program::instruction::{AccountMeta, Instruction}; + use solana_sdk::signature::Keypair; + use super::*; + // -- matching tests (verify our own Blossom impl) -- + #[test] fn max_matching_prioritizes_cardinality() { let edges = vec![(0usize, 1usize, 10_100i32), (1usize, 2usize, 10_090i32)]; @@ -478,6 +565,8 @@ mod tests { assert!(mates.is_empty()); } + // -- pairing helper tests -- + #[test] fn pairing_candidate_count_matches_combination_formula() { assert_eq!(pairing_candidate_count(0), 0); @@ -485,20 +574,16 @@ mod tests { assert_eq!(pairing_candidate_count(2), 1); assert_eq!(pairing_candidate_count(3), 3); assert_eq!(pairing_candidate_count(10), 45); + assert_eq!(pairing_candidate_count(100), 4950); } #[test] fn pairing_precheck_enforces_instruction_and_candidate_limits() { - let max_count_by_candidate_limit = 63; // 63 * 62 / 2 = 1953 assert!(!pairing_precheck_passes(1, pairing_candidate_count(1))); assert!(pairing_precheck_passes(2, pairing_candidate_count(2))); assert!(pairing_precheck_passes( - max_count_by_candidate_limit, - pairing_candidate_count(max_count_by_candidate_limit) - )); - assert!(!pairing_precheck_passes( - max_count_by_candidate_limit + 1, - pairing_candidate_count(max_count_by_candidate_limit + 1) + MAX_PAIRING_INSTRUCTIONS, + pairing_candidate_count(MAX_PAIRING_INSTRUCTIONS) )); assert!(!pairing_precheck_passes( MAX_PAIRING_INSTRUCTIONS + 1, @@ -519,4 +604,234 @@ mod tests { MIN_REMAINING_BLOCKS_FOR_PAIRING + 1 )); } + + // -- transaction size tests -- + + #[test] + fn legacy_transaction_size_is_consistent() { + let payer = Keypair::new(); + let ix = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![AccountMeta::new(payer.pubkey(), true)], + data: vec![0u8; 100], + }; + let tx = Transaction::new_with_payer(&[ix], Some(&payer.pubkey())); + let native_size = legacy_transaction_size(&tx); + // Sanity: a non-trivial tx should be > 200 bytes. + assert!(native_size > 200, "native_size = {native_size}"); + // And under the packet limit. + assert!(native_size < PACKET_DATA_SIZE); + } + + // -- pair_state_nullify_batches integration test -- + + /// Shared test fixtures that mimic real nullify_2 instructions: same + /// program_id, same queue, same merkle tree, differing only in proof + /// remaining-accounts and per-leaf instruction data. + #[allow(dead_code)] + struct TestFixture { + program_id: Pubkey, + authority: Pubkey, + queue: Pubkey, + merkle_tree: Pubkey, + // Base accounts shared by every nullify_2 instruction. + base_accounts: Vec, + } + + impl TestFixture { + fn new(payer: &Keypair) -> Self { + let program_id = Pubkey::new_unique(); + let authority = payer.pubkey(); + let queue = Pubkey::new_unique(); + let merkle_tree = Pubkey::new_unique(); + + // 8 base accounts: authority, forester_pda, registered_program, + // queue, merkle_tree, log_wrapper, cpi_authority, acc_compression + let base_accounts = vec![ + AccountMeta::new(authority, true), + AccountMeta::new(Pubkey::new_unique(), false), + AccountMeta::new_readonly(Pubkey::new_unique(), false), + AccountMeta::new(queue, false), + AccountMeta::new(merkle_tree, false), + AccountMeta::new_readonly(Pubkey::new_unique(), false), + AccountMeta::new_readonly(Pubkey::new_unique(), false), + AccountMeta::new_readonly(Pubkey::new_unique(), false), + ]; + + Self { + program_id, + authority, + queue, + merkle_tree, + base_accounts, + } + } + + fn make_ix( + &self, + leaf_index: u64, + proof_nodes: Vec<[u8; 32]>, + ) -> StateNullifyInstruction { + let mut accounts = self.base_accounts.clone(); + for node in &proof_nodes { + accounts.push(AccountMeta::new_readonly( + Pubkey::new_from_array(*node), + false, + )); + } + let instruction = Instruction { + program_id: self.program_id, + accounts, + data: vec![0u8; 39], // 8-byte discriminator + 31-byte payload + }; + StateNullifyInstruction { + instruction, + proof_nodes, + leaf_index, + merkle_tree: self.merkle_tree, + } + } + } + + fn shared_proof(prefix: u8) -> [u8; 32] { + let mut node = [0u8; 32]; + node[0] = prefix; + node + } + + fn unique_proof(idx: u16) -> [u8; 32] { + let mut node = [0xFFu8; 32]; + node[0] = (idx >> 8) as u8; + node[1] = (idx & 0xFF) as u8; + node + } + + #[test] + fn pair_state_nullify_batches_pairs_overlapping_proofs() { + let payer = Keypair::new(); + let blockhash = Hash::default(); + let fx = TestFixture::new(&payer); + + // 4 instructions, each with exactly 16 proof nodes (realistic). + // ix0 and ix1 share 14/16 nodes (like adjacent leaves in a tree). + // ix2 and ix3 share 14/16 nodes (different subtree). + let shared_0_1: Vec<[u8; 32]> = (0..14).map(shared_proof).collect(); + let shared_2_3: Vec<[u8; 32]> = (100..114).map(shared_proof).collect(); + + let mut proof_0: Vec<[u8; 32]> = shared_0_1.clone(); + proof_0.extend((0..2).map(|i| unique_proof(i))); + let mut proof_1: Vec<[u8; 32]> = shared_0_1; + proof_1.extend((10..12).map(|i| unique_proof(i))); + let mut proof_2: Vec<[u8; 32]> = shared_2_3.clone(); + proof_2.extend((20..22).map(|i| unique_proof(i))); + let mut proof_3: Vec<[u8; 32]> = shared_2_3; + proof_3.extend((40..42).map(|i| unique_proof(i))); + + let ixs = vec![ + fx.make_ix(10, proof_0), + fx.make_ix(11, proof_1), + fx.make_ix(50, proof_2), + fx.make_ix(51, proof_3), + ]; + + let batches = pair_state_nullify_batches( + ixs, + &payer, + &blockhash, + Some(1), + Some(200_000), + false, + ) + .unwrap(); + + // All 4 should be paired into 2 batches. + assert_eq!(batches.len(), 2, "expected 2 paired batches"); + assert_eq!(batches[0].len(), 2, "first batch should have 2 ixs"); + assert_eq!(batches[1].len(), 2, "second batch should have 2 ixs"); + } + + #[test] + fn pair_state_nullify_batches_pairs_only_drops_singles() { + let payer = Keypair::new(); + let blockhash = Hash::default(); + let fx = TestFixture::new(&payer); + + // 3 instructions: ix0 and ix1 share 14/16 proof nodes, ix2 is alone. + let shared: Vec<[u8; 32]> = (0..14).map(shared_proof).collect(); + let mut proof_0: Vec<[u8; 32]> = shared.clone(); + proof_0.extend((0..2).map(|i| unique_proof(i))); + let mut proof_1: Vec<[u8; 32]> = shared; + proof_1.extend((10..12).map(|i| unique_proof(i))); + let proof_2: Vec<[u8; 32]> = (30..46).map(|i| unique_proof(i)).collect(); + + let ixs = vec![ + fx.make_ix(10, proof_0), + fx.make_ix(11, proof_1), + fx.make_ix(90, proof_2), + ]; + + // pairs_only = true → ix2 is dropped. + let batches = pair_state_nullify_batches( + ixs, + &payer, + &blockhash, + Some(1), + Some(200_000), + true, + ) + .unwrap(); + + assert_eq!(batches.len(), 1, "only 1 paired batch expected"); + assert_eq!(batches[0].len(), 2, "the paired batch should have 2 ixs"); + } + + #[test] + fn pair_state_nullify_batches_single_instruction_no_pairs() { + let payer = Keypair::new(); + let blockhash = Hash::default(); + let fx = TestFixture::new(&payer); + + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); + let ixs = vec![fx.make_ix(42, proof)]; + + let batches = pair_state_nullify_batches( + ixs, + &payer, + &blockhash, + Some(1), + Some(200_000), + false, + ) + .unwrap(); + + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].len(), 1); + } + + #[test] + fn pair_state_nullify_batches_sorted_by_leaf_index() { + let payer = Keypair::new(); + let blockhash = Hash::default(); + let fx = TestFixture::new(&payer); + + // Two instructions with identical proofs → will pair. + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); + let ixs = vec![ + fx.make_ix(999, proof.clone()), + fx.make_ix(1, proof), + ]; + + let batches = pair_state_nullify_batches( + ixs, + &payer, + &blockhash, + Some(1), + Some(200_000), + false, + ) + .unwrap(); + + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].len(), 2); + } } diff --git a/forester/src/smart_transaction.rs b/forester/src/smart_transaction.rs index 38df9f70be..c619319812 100644 --- a/forester/src/smart_transaction.rs +++ b/forester/src/smart_transaction.rs @@ -173,7 +173,7 @@ pub fn collect_priority_fee_accounts(payer: Pubkey, instructions: &[Instruction] account_keys } -fn with_compute_budget_instructions( +pub(crate) fn with_compute_budget_instructions( mut instructions: Vec, compute_budget: ComputeBudgetConfig, ) -> Vec { diff --git a/program-tests/registry-test/tests/nullify_2_regression.rs b/program-tests/registry-test/tests/nullify_2_regression.rs index ef6fdfbb5f..d35b3846a3 100644 --- a/program-tests/registry-test/tests/nullify_2_regression.rs +++ b/program-tests/registry-test/tests/nullify_2_regression.rs @@ -120,7 +120,7 @@ async fn test_nullify_2_validation_and_success() { let result = rpc .create_and_send_transaction(&[empty_proof_accounts_ix], &forester.pubkey(), &[&forester]) .await; - assert_rpc_error(result, 0, RegistryError::EmptyProofAccounts.into()).unwrap(); + assert_rpc_error(result, 0, RegistryError::InvalidProofAccountsLength.into()).unwrap(); let malformed_ix = Instruction { program_id: light_registry::ID, diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index 5592b21cb0..c936f6e216 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -82,9 +82,6 @@ pub(crate) fn validate_nullify_2_inputs( if change_log_indices.len() != 1 || leaves_queue_indices.len() != 1 || indices.len() != 1 { return err!(RegistryError::InvalidNullify2Inputs); } - if proof_accounts_len == 0 { - return err!(RegistryError::EmptyProofAccounts); - } if proof_accounts_len != NULLIFY_2_PROOF_ACCOUNTS_LEN { return err!(RegistryError::InvalidProofAccountsLength); } @@ -107,7 +104,7 @@ mod tests { let result = validate_nullify_2_inputs(&[1], &[1], &[42], 0); assert_eq!( result.err().unwrap(), - RegistryError::EmptyProofAccounts.into() + RegistryError::InvalidProofAccountsLength.into() ); } diff --git a/programs/registry/src/errors.rs b/programs/registry/src/errors.rs index a47293e823..73384abb41 100644 --- a/programs/registry/src/errors.rs +++ b/programs/registry/src/errors.rs @@ -38,8 +38,6 @@ pub enum RegistryError { InvalidTokenAccountData, #[msg("Indices array cannot be empty")] EmptyIndices, - #[msg("Proof accounts cannot be empty")] - EmptyProofAccounts, #[msg("Nullify2 proof accounts length is invalid")] InvalidProofAccountsLength, #[msg("Nullify2 supports exactly one change, queue index, and leaf index")] From b8d234b7b7845132e45184488b3bf627e8252b2e Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Tue, 17 Mar 2026 18:29:58 +0000 Subject: [PATCH 12/12] Refactor nullify_2 functionality and improve transaction handling - Enhanced the transaction building process in `tx_builder.rs` to support paired nullify_2 instructions, improving efficiency. - Updated the instruction batching logic to separate address updates from state nullifications, allowing for better transaction management. - Introduced a new method for estimating transaction sizes without cloning instructions, optimizing performance. - Modified the nullify_2 instruction validation to streamline input checks and ensure proper proof account lengths. - Improved test coverage for nullify_2 functionality, including tests for paired transactions and validation scenarios. - Removed redundant validation functions and adjusted error handling for clarity and consistency. --- forester/docs/v1_forester_flows.md | 2 +- forester/src/epoch_manager.rs | 2 +- forester/src/lib.rs | 2 +- forester/src/matching.rs | 41 +- forester/src/processor/v1/config.rs | 10 +- forester/src/processor/v1/send_transaction.rs | 45 +- forester/src/processor/v1/tx_builder.rs | 404 ++++++++++++------ .../tests/nullify_2_regression.rs | 274 ++++++------ .../src/account_compression_cpi/nullify.rs | 58 +-- .../src/account_compression_cpi/sdk.rs | 10 +- programs/registry/src/errors.rs | 6 +- programs/registry/src/lib.rs | 23 +- 12 files changed, 509 insertions(+), 368 deletions(-) diff --git a/forester/docs/v1_forester_flows.md b/forester/docs/v1_forester_flows.md index 8f1c77489f..a3d9b2ddcb 100644 --- a/forester/docs/v1_forester_flows.md +++ b/forester/docs/v1_forester_flows.md @@ -111,7 +111,7 @@ └── YES → pair_state_nullify_batches │ │ For each pair (i,j): - │ - pair_fits_transaction_size(ix_i, ix_j)? (serialized <= 1232) + │ - estimated_tx_size(ix_i, ix_j) <= 1200? (packet - safety margin) │ - weight = 10000 + proof_overlap_count │ │ Max-cardinality matching (mwmatching) diff --git a/forester/src/epoch_manager.rs b/forester/src/epoch_manager.rs index 1f4e83dfc5..96abb8b01d 100644 --- a/forester/src/epoch_manager.rs +++ b/forester/src/epoch_manager.rs @@ -2993,7 +2993,7 @@ impl EpochManager { compute_unit_limit: Some(self.config.transaction_config.cu_limit), enable_priority_fees: self.config.transaction_config.enable_priority_fees, max_concurrent_sends: Some(self.config.transaction_config.max_concurrent_sends), - pairs_only: false, + pairs_only: false, // overridden at runtime based on queue fullness }, queue_config: self.config.queue_config, retry_config: RetryConfig { diff --git a/forester/src/lib.rs b/forester/src/lib.rs index 546e43cf7e..264b2066ea 100644 --- a/forester/src/lib.rs +++ b/forester/src/lib.rs @@ -5,12 +5,12 @@ pub mod cli; pub mod compressible; pub mod config; pub mod epoch_manager; -pub(crate) mod matching; pub mod errors; pub mod forester_status; pub mod health_check; pub mod helius_priority_fee_types; pub mod logging; +pub(crate) mod matching; pub mod metrics; pub mod pagerduty; pub mod priority_fee; diff --git a/forester/src/matching.rs b/forester/src/matching.rs index 0bdc6ef3db..d108bed7c4 100644 --- a/forester/src/matching.rs +++ b/forester/src/matching.rs @@ -161,6 +161,7 @@ fn find_lca( } /// Shrink the blossom defined by paths v→lca and u→lca. +#[allow(clippy::too_many_arguments)] fn contract( base: &mut [usize], parent: &mut [usize], @@ -234,7 +235,13 @@ mod tests { let pairs: Vec<_> = mates .iter() .enumerate() - .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .filter_map(|(i, &m)| { + if m != SENTINEL && m > i { + Some((i, m)) + } else { + None + } + }) .collect(); assert_eq!(pairs.len(), 1); } @@ -271,7 +278,13 @@ mod tests { let pairs: Vec<_> = mates .iter() .enumerate() - .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .filter_map(|(i, &m)| { + if m != SENTINEL && m > i { + Some((i, m)) + } else { + None + } + }) .collect(); assert_eq!(pairs.len(), 2); } @@ -284,7 +297,13 @@ mod tests { let pairs: Vec<_> = mates .iter() .enumerate() - .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .filter_map(|(i, &m)| { + if m != SENTINEL && m > i { + Some((i, m)) + } else { + None + } + }) .collect(); assert_eq!(pairs.len(), 2); } @@ -302,7 +321,13 @@ mod tests { let pairs: Vec<_> = mates .iter() .enumerate() - .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .filter_map(|(i, &m)| { + if m != SENTINEL && m > i { + Some((i, m)) + } else { + None + } + }) .collect(); assert_eq!(pairs.len(), 3); } @@ -318,7 +343,13 @@ mod tests { let pairs: Vec<_> = mates .iter() .enumerate() - .filter_map(|(i, &m)| if m != SENTINEL && m > i { Some((i, m)) } else { None }) + .filter_map(|(i, &m)| { + if m != SENTINEL && m > i { + Some((i, m)) + } else { + None + } + }) .collect(); assert_eq!(pairs.len(), 2); // perfect matching } diff --git a/forester/src/processor/v1/config.rs b/forester/src/processor/v1/config.rs index bba3db3a86..153e005c96 100644 --- a/forester/src/processor/v1/config.rs +++ b/forester/src/processor/v1/config.rs @@ -21,6 +21,12 @@ pub struct SendBatchedTransactionsConfig { pub confirmation_max_attempts: usize, } +/// Pending-item threshold below which the forester only emits *paired* +/// state-nullify transactions, dropping unpaired singles. When the queue is +/// nearly empty there is no urgency, so we save a transaction by waiting for +/// the next cycle when the single can potentially be paired. +pub const PAIRS_ONLY_THRESHOLD: u64 = 4_000; + #[derive(Debug, Clone, Copy)] pub struct BuildTransactionBatchConfig { pub batch_size: u64, @@ -29,7 +35,7 @@ pub struct BuildTransactionBatchConfig { pub enable_priority_fees: bool, pub max_concurrent_sends: Option, /// When `true`, only emit paired state-nullify transactions. - /// Unpaired (single) nullifies are dropped. The caller sets this based - /// on queue fullness: `pairs_only = queue_pending < MAX_QUEUE_FULLNESS`. + /// Unpaired singles are dropped and retried in the next cycle. + /// Computed at runtime: `pairs_only = total_pending < PAIRS_ONLY_THRESHOLD`. pub pairs_only: bool, } diff --git a/forester/src/processor/v1/send_transaction.rs b/forester/src/processor/v1/send_transaction.rs index b5282bc47a..9eddc83889 100644 --- a/forester/src/processor/v1/send_transaction.rs +++ b/forester/src/processor/v1/send_transaction.rs @@ -27,7 +27,9 @@ use crate::{ errors::ForesterError, metrics::increment_transactions_failed, priority_fee::PriorityFeeConfig, - processor::v1::{config::SendBatchedTransactionsConfig, tx_builder::TransactionBuilder}, + processor::v1::{ + config, config::SendBatchedTransactionsConfig, tx_builder::TransactionBuilder, + }, queue_helpers::fetch_queue_item_data, smart_transaction::{ConfirmationConfig, PreparedTransaction, SmartTransactionError}, Result, @@ -39,6 +41,7 @@ struct PreparedBatchData { last_valid_block_height: u64, priority_fee: Option, timeout_deadline: Instant, + total_pending: u64, } #[derive(Clone)] @@ -97,22 +100,24 @@ pub async fn send_batched_transactions( } }; - let queue_item_data = { + let (queue_item_data, total_pending) = { let mut rpc = pool.get_connection().await.map_err(|e| { error!(tree = %tree_id_str, "Failed to get RPC for queue data: {:?}", e); ForesterError::RpcPool(e) })?; - fetch_queue_item_data(&mut *rpc, &tree_accounts.queue, queue_fetch_start_index) - .await - .map_err(|e| { - warn!(tree = %tree_id_str, "Failed to fetch queue item data: {:?}", e); - ForesterError::General { - error: format!("Fetch queue data failed for {}: {}", tree_id_str, e), - } - })? - .items + let result = + fetch_queue_item_data(&mut *rpc, &tree_accounts.queue, queue_fetch_start_index) + .await + .map_err(|e| { + warn!(tree = %tree_id_str, "Failed to fetch queue item data: {:?}", e); + ForesterError::General { + error: format!("Fetch queue data failed for {}: {}", tree_id_str, e), + } + })?; + (result.items, result.total_pending) }; if queue_item_data.is_empty() { @@ -315,6 +321,7 @@ async fn prepare_batch_prerequisites( last_valid_block_height, priority_fee, timeout_deadline, + total_pending, })) } diff --git a/forester/src/processor/v1/tx_builder.rs b/forester/src/processor/v1/tx_builder.rs index 4b5d495860..17b267a67e 100644 --- a/forester/src/processor/v1/tx_builder.rs +++ b/forester/src/processor/v1/tx_builder.rs @@ -6,11 +6,12 @@ use forester_utils::rpc_pool::SolanaRpcPool; use light_client::rpc::Rpc; use solana_program::hash::Hash; use solana_sdk::{ + compute_budget::ComputeBudgetInstruction, signature::{Keypair, Signer}, transaction::Transaction, }; use tokio::sync::Mutex; -use tracing::{trace, warn}; +use tracing::{info, trace, warn}; use crate::{ epoch_manager::WorkItem, @@ -25,10 +26,7 @@ use crate::{ }, }, }, - smart_transaction::{ - create_smart_transaction, with_compute_budget_instructions, ComputeBudgetConfig, - CreateSmartTransactionConfig, - }, + smart_transaction::{create_smart_transaction, CreateSmartTransactionConfig}, Result, }; @@ -219,7 +217,7 @@ impl TransactionBuilder for EpochManagerTransactions { .iter() .filter(|ix| matches!(ix, PreparedV1Instruction::StateNullify(_))) .count(); - let allow_pairing = if batch_size >= 2 { + let allow_pairing = if state_nullify_count >= 2 { self.should_attempt_pairing(last_valid_block_height, state_nullify_count) .await } else { @@ -230,13 +228,13 @@ impl TransactionBuilder for EpochManagerTransactions { batch_size, allow_pairing, config.pairs_only, - payer, - recent_blockhash, + &payer.pubkey(), priority_fee, config.compute_unit_limit, )?; for instruction_chunk in instruction_batches { + let is_paired = instruction_chunk.len() >= 2; let (transaction, _) = create_smart_transaction(CreateSmartTransactionConfig { payer: payer.insecure_clone(), instructions: instruction_chunk, @@ -246,6 +244,16 @@ impl TransactionBuilder for EpochManagerTransactions { last_valid_block_height, }) .await?; + if is_paired { + info!( + "Paired nullify_2 tx: sig={}, ixs=2", + transaction + .signatures + .first() + .map(|s| s.to_string()) + .unwrap_or_default() + ); + } transactions.push(transaction); } @@ -274,8 +282,7 @@ fn build_instruction_batches( batch_size: usize, allow_pairing: bool, pairs_only: bool, - payer: &Keypair, - recent_blockhash: &Hash, + payer: &Pubkey, priority_fee: Option, compute_unit_limit: Option, ) -> Result>> { @@ -300,11 +307,10 @@ fn build_instruction_batches( // Sort by leaf_index for better proof-node overlap between neighbours. state_nullify_instructions.sort_by_key(|ix| ix.leaf_index); - let paired_batches = if batch_size >= 2 && allow_pairing { + let paired_batches = if allow_pairing { pair_state_nullify_batches( state_nullify_instructions, payer, - recent_blockhash, priority_fee, compute_unit_limit, pairs_only, @@ -323,8 +329,7 @@ fn build_instruction_batches( fn pair_state_nullify_batches( state_nullify_instructions: Vec, - payer: &Keypair, - recent_blockhash: &Hash, + payer: &Pubkey, priority_fee: Option, compute_unit_limit: Option, pairs_only: bool, @@ -340,6 +345,9 @@ fn pair_state_nullify_batches( .collect()); } + // Pre-compute compute budget instructions once for all pairs. + let compute_budget_ixs = make_compute_budget_instructions(priority_fee, compute_unit_limit); + // Pre-compute HashSets for O(1) overlap lookup. let proof_sets: Vec> = state_nullify_instructions .iter() @@ -353,14 +361,15 @@ fn pair_state_nullify_batches( let mut edges: Vec<(usize, usize, i32)> = Vec::new(); for i in 0..n { for j in (i + 1)..n { - if !pair_fits_transaction_size( - &state_nullify_instructions[i].instruction, - &state_nullify_instructions[j].instruction, + if estimated_tx_size( payer, - recent_blockhash, - priority_fee, - compute_unit_limit, - ) { + &compute_budget_ixs, + &[ + &state_nullify_instructions[i].instruction, + &state_nullify_instructions[j].instruction, + ], + ) > MAX_TRANSACTION_SIZE + { continue; } let overlap = proof_sets[i].intersection(&proof_sets[j]).count() as i32; @@ -432,37 +441,59 @@ fn pair_state_nullify_batches( } // --------------------------------------------------------------------------- -// Transaction-size estimation (no bincode – native wire-format calculation) +// Transaction-size estimation (zero-copy – no instruction cloning) // --------------------------------------------------------------------------- -/// Check whether two instructions plus compute-budget prefixes fit inside a -/// single Solana legacy transaction. Uses the same construction path as -/// [`create_smart_transaction`] to avoid divergence. -fn pair_fits_transaction_size( - ix_a: &solana_program::instruction::Instruction, - ix_b: &solana_program::instruction::Instruction, - payer: &Keypair, - _recent_blockhash: &Hash, +/// Build the compute-budget instructions that `create_smart_transaction` would +/// prepend. Built once and reused across all pair checks. +fn make_compute_budget_instructions( priority_fee: Option, compute_unit_limit: Option, -) -> bool { - // Build instructions exactly as create_smart_transaction does. - let final_instructions = with_compute_budget_instructions( - vec![ix_a.clone(), ix_b.clone()], - ComputeBudgetConfig { - compute_unit_price: priority_fee, - compute_unit_limit, - }, - ); - let tx = Transaction::new_with_payer(&final_instructions, Some(&payer.pubkey())); - legacy_transaction_size(&tx) <= MAX_TRANSACTION_SIZE +) -> Vec { + let mut ixs = Vec::with_capacity(2); + if let Some(price) = priority_fee { + ixs.push(ComputeBudgetInstruction::set_compute_unit_price(price)); + } + if let Some(limit) = compute_unit_limit { + ixs.push(ComputeBudgetInstruction::set_compute_unit_limit(limit)); + } + ixs } -/// Compute the Solana legacy-transaction wire-format size without pulling in -/// a serialisation crate. -fn legacy_transaction_size(tx: &Transaction) -> usize { - let msg = &tx.message; - let num_sigs = msg.header.num_required_signatures as usize; +/// Estimate the Solana legacy-transaction wire-format size from instruction +/// references, without cloning instructions or constructing a Transaction. +fn estimated_tx_size( + payer: &Pubkey, + compute_budget_ixs: &[solana_program::instruction::Instruction], + main_ixs: &[&solana_program::instruction::Instruction], +) -> usize { + let mut keys = HashSet::new(); + keys.insert(*payer); + + let mut signer_keys = HashSet::new(); + signer_keys.insert(*payer); + + for ix in compute_budget_ixs { + keys.insert(ix.program_id); + for meta in &ix.accounts { + keys.insert(meta.pubkey); + if meta.is_signer { + signer_keys.insert(meta.pubkey); + } + } + } + for ix in main_ixs { + keys.insert(ix.program_id); + for meta in &ix.accounts { + keys.insert(meta.pubkey); + if meta.is_signer { + signer_keys.insert(meta.pubkey); + } + } + } + + let num_keys = keys.len(); + let num_sigs = signer_keys.len(); // signatures section: compact-u16(count) + count * 64 let sigs = short_vec_len(num_sigs) + num_sigs * 64; @@ -471,15 +502,43 @@ fn legacy_transaction_size(tx: &Transaction) -> usize { let header = 3; // account keys: compact-u16(count) + count * 32 - let keys = short_vec_len(msg.account_keys.len()) + msg.account_keys.len() * 32; + let key_bytes = short_vec_len(num_keys) + num_keys * 32; // recent_blockhash let blockhash = 32; // instructions: compact-u16(count) + each instruction + let instruction_count = compute_budget_ixs.len() + main_ixs.len(); + let mut ixs = short_vec_len(instruction_count); + for ix in compute_budget_ixs { + ixs += 1; // program_id_index (u8) + ixs += short_vec_len(ix.accounts.len()) + ix.accounts.len(); + ixs += short_vec_len(ix.data.len()) + ix.data.len(); + } + for ix in main_ixs { + ixs += 1; + ixs += short_vec_len(ix.accounts.len()) + ix.accounts.len(); + ixs += short_vec_len(ix.data.len()) + ix.data.len(); + } + + sigs + header + key_bytes + blockhash + ixs +} + +/// Compute the Solana legacy-transaction wire-format size from a constructed +/// Transaction. Used in tests to verify `estimated_tx_size` correctness. +#[cfg(test)] +fn legacy_transaction_size(tx: &Transaction) -> usize { + let msg = &tx.message; + let num_sigs = msg.header.num_required_signatures as usize; + + let sigs = short_vec_len(num_sigs) + num_sigs * 64; + let header = 3; + let keys = short_vec_len(msg.account_keys.len()) + msg.account_keys.len() * 32; + let blockhash = 32; + let mut ixs = short_vec_len(msg.instructions.len()); for ix in &msg.instructions { - ixs += 1; // program_id_index (u8) + ixs += 1; ixs += short_vec_len(ix.accounts.len()) + ix.accounts.len(); ixs += short_vec_len(ix.data.len()) + ix.data.len(); } @@ -607,6 +666,59 @@ mod tests { // -- transaction size tests -- + #[test] + fn estimated_tx_size_matches_legacy_transaction_size() { + let payer = Keypair::new(); + let program_id = Pubkey::new_unique(); + let ix = Instruction { + program_id, + accounts: vec![AccountMeta::new(payer.pubkey(), true)], + data: vec![0u8; 100], + }; + let compute_budget_ixs = make_compute_budget_instructions(Some(1_000), Some(200_000)); + + // Estimate without constructing a transaction. + let estimated = estimated_tx_size(&payer.pubkey(), &compute_budget_ixs, &[&ix]); + + // Build the real transaction for comparison. + let mut all_ixs = compute_budget_ixs; + all_ixs.push(ix); + let tx = Transaction::new_with_payer(&all_ixs, Some(&payer.pubkey())); + let actual = legacy_transaction_size(&tx); + + assert_eq!(estimated, actual); + } + + #[test] + fn estimated_tx_size_with_two_instructions() { + let payer = Keypair::new(); + let fx = TestFixture::new(&payer); + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); + let ix_a = fx.make_ix(10, proof.clone()); + let ix_b = fx.make_ix(11, proof); + let compute_budget_ixs = make_compute_budget_instructions(Some(1), Some(200_000)); + + let estimated = estimated_tx_size( + &payer.pubkey(), + &compute_budget_ixs, + &[&ix_a.instruction, &ix_b.instruction], + ); + + // Build the real transaction for comparison. + let mut all_ixs = compute_budget_ixs; + all_ixs.push(ix_a.instruction); + all_ixs.push(ix_b.instruction); + let tx = Transaction::new_with_payer(&all_ixs, Some(&payer.pubkey())); + let actual = legacy_transaction_size(&tx); + + assert_eq!(estimated, actual); + // Two nullify_2 instructions with 16 shared proof accounts should fit. + assert!( + estimated <= MAX_TRANSACTION_SIZE, + "estimated={estimated} > MAX_TRANSACTION_SIZE={MAX_TRANSACTION_SIZE}" + ); + } + #[test] fn legacy_transaction_size_is_consistent() { let payer = Keypair::new(); @@ -623,16 +735,13 @@ mod tests { assert!(native_size < PACKET_DATA_SIZE); } - // -- pair_state_nullify_batches integration test -- + // -- pair_state_nullify_batches integration tests -- /// Shared test fixtures that mimic real nullify_2 instructions: same /// program_id, same queue, same merkle tree, differing only in proof /// remaining-accounts and per-leaf instruction data. - #[allow(dead_code)] struct TestFixture { program_id: Pubkey, - authority: Pubkey, - queue: Pubkey, merkle_tree: Pubkey, // Base accounts shared by every nullify_2 instruction. base_accounts: Vec, @@ -641,14 +750,13 @@ mod tests { impl TestFixture { fn new(payer: &Keypair) -> Self { let program_id = Pubkey::new_unique(); - let authority = payer.pubkey(); let queue = Pubkey::new_unique(); let merkle_tree = Pubkey::new_unique(); // 8 base accounts: authority, forester_pda, registered_program, // queue, merkle_tree, log_wrapper, cpi_authority, acc_compression let base_accounts = vec![ - AccountMeta::new(authority, true), + AccountMeta::new(payer.pubkey(), true), AccountMeta::new(Pubkey::new_unique(), false), AccountMeta::new_readonly(Pubkey::new_unique(), false), AccountMeta::new(queue, false), @@ -660,18 +768,12 @@ mod tests { Self { program_id, - authority, - queue, merkle_tree, base_accounts, } } - fn make_ix( - &self, - leaf_index: u64, - proof_nodes: Vec<[u8; 32]>, - ) -> StateNullifyInstruction { + fn make_ix(&self, leaf_index: u64, proof_nodes: Vec<[u8; 32]>) -> StateNullifyInstruction { let mut accounts = self.base_accounts.clone(); for node in &proof_nodes { accounts.push(AccountMeta::new_readonly( @@ -682,7 +784,7 @@ mod tests { let instruction = Instruction { program_id: self.program_id, accounts, - data: vec![0u8; 39], // 8-byte discriminator + 31-byte payload + data: vec![0u8; 27], // 8-byte discriminator + 19-byte scalar payload }; StateNullifyInstruction { instruction, @@ -709,7 +811,6 @@ mod tests { #[test] fn pair_state_nullify_batches_pairs_overlapping_proofs() { let payer = Keypair::new(); - let blockhash = Hash::default(); let fx = TestFixture::new(&payer); // 4 instructions, each with exactly 16 proof nodes (realistic). @@ -719,13 +820,13 @@ mod tests { let shared_2_3: Vec<[u8; 32]> = (100..114).map(shared_proof).collect(); let mut proof_0: Vec<[u8; 32]> = shared_0_1.clone(); - proof_0.extend((0..2).map(|i| unique_proof(i))); + proof_0.extend((0..2).map(unique_proof)); let mut proof_1: Vec<[u8; 32]> = shared_0_1; - proof_1.extend((10..12).map(|i| unique_proof(i))); + proof_1.extend((10..12).map(unique_proof)); let mut proof_2: Vec<[u8; 32]> = shared_2_3.clone(); - proof_2.extend((20..22).map(|i| unique_proof(i))); + proof_2.extend((20..22).map(unique_proof)); let mut proof_3: Vec<[u8; 32]> = shared_2_3; - proof_3.extend((40..42).map(|i| unique_proof(i))); + proof_3.extend((40..42).map(unique_proof)); let ixs = vec![ fx.make_ix(10, proof_0), @@ -734,15 +835,9 @@ mod tests { fx.make_ix(51, proof_3), ]; - let batches = pair_state_nullify_batches( - ixs, - &payer, - &blockhash, - Some(1), - Some(200_000), - false, - ) - .unwrap(); + let batches = + pair_state_nullify_batches(ixs, &payer.pubkey(), Some(1), Some(200_000), false) + .unwrap(); // All 4 should be paired into 2 batches. assert_eq!(batches.len(), 2, "expected 2 paired batches"); @@ -751,87 +846,146 @@ mod tests { } #[test] - fn pair_state_nullify_batches_pairs_only_drops_singles() { + fn pair_state_nullify_batches_single_instruction_no_pairs() { let payer = Keypair::new(); - let blockhash = Hash::default(); let fx = TestFixture::new(&payer); - // 3 instructions: ix0 and ix1 share 14/16 proof nodes, ix2 is alone. - let shared: Vec<[u8; 32]> = (0..14).map(shared_proof).collect(); - let mut proof_0: Vec<[u8; 32]> = shared.clone(); - proof_0.extend((0..2).map(|i| unique_proof(i))); - let mut proof_1: Vec<[u8; 32]> = shared; - proof_1.extend((10..12).map(|i| unique_proof(i))); - let proof_2: Vec<[u8; 32]> = (30..46).map(|i| unique_proof(i)).collect(); + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); + let ixs = vec![fx.make_ix(42, proof)]; - let ixs = vec![ - fx.make_ix(10, proof_0), - fx.make_ix(11, proof_1), - fx.make_ix(90, proof_2), - ]; + let batches = + pair_state_nullify_batches(ixs, &payer.pubkey(), Some(1), Some(200_000), false) + .unwrap(); - // pairs_only = true → ix2 is dropped. - let batches = pair_state_nullify_batches( - ixs, - &payer, - &blockhash, - Some(1), - Some(200_000), - true, - ) - .unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].len(), 1); + } + + #[test] + fn pair_state_nullify_batches_sorted_by_leaf_index() { + let payer = Keypair::new(); + let fx = TestFixture::new(&payer); + + // Two instructions with identical proofs → will pair. + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); + let ixs = vec![fx.make_ix(999, proof.clone()), fx.make_ix(1, proof)]; + + let batches = + pair_state_nullify_batches(ixs, &payer.pubkey(), Some(1), Some(200_000), false) + .unwrap(); - assert_eq!(batches.len(), 1, "only 1 paired batch expected"); - assert_eq!(batches[0].len(), 2, "the paired batch should have 2 ixs"); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].len(), 2); } #[test] - fn pair_state_nullify_batches_single_instruction_no_pairs() { + fn pair_state_nullify_batches_no_edges_falls_back_to_singles() { + let payer = Keypair::new(); + let fx = TestFixture::new(&payer); + + // Create instructions with huge data that won't fit paired in one tx. + let make_big_ix = |leaf_index: u64| -> StateNullifyInstruction { + let proof_nodes: Vec<[u8; 32]> = (0..16) + .map(|i| unique_proof(leaf_index as u16 * 100 + i)) + .collect(); + let mut accounts = fx.base_accounts.clone(); + for node in &proof_nodes { + accounts.push(AccountMeta::new_readonly( + Pubkey::new_from_array(*node), + false, + )); + } + // Large data payload to force tx over size limit when paired. + let instruction = Instruction { + program_id: fx.program_id, + accounts, + data: vec![0u8; 500], + }; + StateNullifyInstruction { + instruction, + proof_nodes, + leaf_index, + merkle_tree: fx.merkle_tree, + } + }; + + let ixs = vec![make_big_ix(1), make_big_ix(2)]; + let batches = + pair_state_nullify_batches(ixs, &payer.pubkey(), Some(1), Some(200_000), false) + .unwrap(); + + // Both should be singles since pairing exceeds tx size. + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].len(), 1); + assert_eq!(batches[1].len(), 1); + } + + #[test] + fn build_instruction_batches_separates_address_and_state() { let payer = Keypair::new(); - let blockhash = Hash::default(); let fx = TestFixture::new(&payer); + let addr_ix = Instruction { + program_id: Pubkey::new_unique(), + accounts: vec![AccountMeta::new(payer.pubkey(), true)], + data: vec![0u8; 50], + }; + let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); - let ixs = vec![fx.make_ix(42, proof)]; + let state_ix_0 = fx.make_ix(10, proof.clone()); + let state_ix_1 = fx.make_ix(11, proof); + + let prepared = vec![ + PreparedV1Instruction::AddressUpdate(addr_ix), + PreparedV1Instruction::StateNullify(state_ix_0), + PreparedV1Instruction::StateNullify(state_ix_1), + ]; - let batches = pair_state_nullify_batches( - ixs, - &payer, - &blockhash, + let batches = build_instruction_batches( + prepared, + 2, + true, + false, + &payer.pubkey(), Some(1), Some(200_000), - false, ) .unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0].len(), 1); + // 1 address batch + 1 paired state batch. + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].len(), 1, "address batch should have 1 ix"); + assert_eq!(batches[1].len(), 2, "state batch should be paired"); } #[test] - fn pair_state_nullify_batches_sorted_by_leaf_index() { + fn build_instruction_batches_no_pairing_when_disabled() { let payer = Keypair::new(); - let blockhash = Hash::default(); let fx = TestFixture::new(&payer); - // Two instructions with identical proofs → will pair. let proof: Vec<[u8; 32]> = (0..16).map(shared_proof).collect(); - let ixs = vec![ - fx.make_ix(999, proof.clone()), - fx.make_ix(1, proof), + let state_ix_0 = fx.make_ix(10, proof.clone()); + let state_ix_1 = fx.make_ix(11, proof); + + let prepared = vec![ + PreparedV1Instruction::StateNullify(state_ix_0), + PreparedV1Instruction::StateNullify(state_ix_1), ]; - let batches = pair_state_nullify_batches( - ixs, - &payer, - &blockhash, + let batches = build_instruction_batches( + prepared, + 2, + false, // pairing disabled + false, + &payer.pubkey(), Some(1), Some(200_000), - false, ) .unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0].len(), 2); + // Each state nullify should be a separate batch. + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].len(), 1); + assert_eq!(batches[1].len(), 1); } } diff --git a/program-tests/registry-test/tests/nullify_2_regression.rs b/program-tests/registry-test/tests/nullify_2_regression.rs index d35b3846a3..465248d340 100644 --- a/program-tests/registry-test/tests/nullify_2_regression.rs +++ b/program-tests/registry-test/tests/nullify_2_regression.rs @@ -1,10 +1,10 @@ use account_compression::state::QueueAccount; -use anchor_lang::InstructionData; use forester_utils::account_zero_copy::{get_concurrent_merkle_tree, get_hash_set}; use light_compressed_account::TreeType; use light_hasher::Poseidon; use light_program_test::{ - program_test::LightProgramTest, utils::assert::assert_rpc_error, ProgramTestConfig, + indexer::state_tree::StateMerkleTreeBundle, program_test::LightProgramTest, + utils::assert::assert_rpc_error, ProgramTestConfig, }; use light_registry::{ account_compression_cpi::sdk::{ @@ -16,27 +16,37 @@ use light_registry::{ use light_test_utils::{e2e_test_env::init_program_test_env, Rpc}; use serial_test::serial; use solana_sdk::{ - instruction::Instruction, pubkey::Pubkey, signature::{Keypair, Signer}, }; -#[serial] -#[tokio::test] -async fn test_nullify_2_validation_and_success() { +/// Queue item data extracted from the nullifier queue. +struct QueueEntry { + queue_index: u16, + leaf_index: u64, + proof: Vec<[u8; 32]>, + change_log_index: u64, +} + +/// Shared test environment: creates a state tree, compresses SOL, and transfers +/// to populate the nullifier queue. +async fn setup_tree_with_nullifier_queue_entries( + num_entries: usize, +) -> (LightProgramTest, StateMerkleTreeBundle, Keypair) { let mut rpc = LightProgramTest::new(ProgramTestConfig::default_with_batched_trees(true)) .await .unwrap(); rpc.indexer = None; let env = rpc.test_accounts.clone(); let forester = Keypair::new(); - rpc.airdrop_lamports(&forester.pubkey(), 1_000_000_000) + rpc.airdrop_lamports(&forester.pubkey(), 10_000_000_000) .await .unwrap(); let merkle_tree_keypair = Keypair::new(); let nullifier_queue_keypair = Keypair::new(); let cpi_context_keypair = Keypair::new(); - let (mut rpc, state_tree_bundle) = { + + let (rpc, state_tree_bundle) = { let mut e2e_env = init_program_test_env(rpc, &env, 50).await; e2e_env.indexer.state_merkle_trees.clear(); e2e_env.keypair_action_config.fee_assert = false; @@ -52,92 +62,106 @@ async fn test_nullify_2_validation_and_success() { TreeType::StateV1, ) .await; - e2e_env - .compress_sol_deterministic(&forester, 1_000_000, None) - .await; - e2e_env - .transfer_sol_deterministic(&forester, &Pubkey::new_unique(), None) - .await - .unwrap(); + + for _ in 0..num_entries { + e2e_env + .compress_sol_deterministic(&forester, 1_000_000, None) + .await; + e2e_env + .transfer_sol_deterministic(&forester, &Pubkey::new_unique(), None) + .await + .unwrap(); + } + (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) }; + (rpc, state_tree_bundle, forester) +} + +/// Read pending queue entries from the nullifier queue. +async fn read_queue_entries( + rpc: &mut LightProgramTest, + state_tree_bundle: &StateMerkleTreeBundle, + max_entries: usize, +) -> Vec { let nullifier_queue = unsafe { - get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await + get_hash_set::(rpc, state_tree_bundle.accounts.nullifier_queue).await } .unwrap(); - let mut queue_index = None; - let mut account_hash = None; - for i in 0..nullifier_queue.get_capacity() { - let bucket = nullifier_queue.get_bucket(i).unwrap(); - if let Some(bucket) = bucket { - if bucket.sequence_number.is_none() { - queue_index = Some(i as u16); - account_hash = Some(bucket.value_bytes()); - break; - } - } - } - let queue_index = queue_index.unwrap(); - let account_hash = account_hash.unwrap(); - let leaf_index = state_tree_bundle - .merkle_tree - .get_leaf_index(&account_hash) - .unwrap() as u64; - let proof = state_tree_bundle - .merkle_tree - .get_proof_of_leaf(leaf_index as usize, false) - .unwrap(); - let proof_depth = proof.len(); + let onchain_tree = get_concurrent_merkle_tree::( - &mut rpc, + rpc, state_tree_bundle.accounts.merkle_tree, ) .await .unwrap(); let change_log_index = onchain_tree.changelog_index() as u64; + let mut entries = Vec::new(); + for i in 0..nullifier_queue.get_capacity() { + if entries.len() >= max_entries { + break; + } + let bucket = nullifier_queue.get_bucket(i).unwrap(); + if let Some(bucket) = bucket { + if bucket.sequence_number.is_none() { + let account_hash = bucket.value_bytes(); + let leaf_index = state_tree_bundle + .merkle_tree + .get_leaf_index(&account_hash) + .unwrap() as u64; + let proof = state_tree_bundle + .merkle_tree + .get_proof_of_leaf(leaf_index as usize, false) + .unwrap(); + + entries.push(QueueEntry { + queue_index: i as u16, + leaf_index, + proof, + change_log_index, + }); + } + } + } + entries +} + +#[serial] +#[tokio::test] +async fn test_nullify_2_validation_and_success() { + let (mut rpc, state_tree_bundle, forester) = setup_tree_with_nullifier_queue_entries(1).await; + let entries = read_queue_entries(&mut rpc, &state_tree_bundle, 1).await; + let entry = &entries[0]; + let valid_ix = create_nullify_2_instruction( CreateNullify2InstructionInputs { authority: forester.pubkey(), nullifier_queue: state_tree_bundle.accounts.nullifier_queue, merkle_tree: state_tree_bundle.accounts.merkle_tree, - change_log_index, - leaves_queue_index: queue_index, - index: leaf_index, - proof: proof.try_into().unwrap(), + change_log_index: entry.change_log_index, + leaves_queue_index: entry.queue_index, + index: entry.leaf_index, + proof: entry.proof.clone().try_into().unwrap(), derivation: forester.pubkey(), is_metadata_forester: true, }, 0, ); + // Test: empty proof accounts → InvalidProofAccountsLength. let mut empty_proof_accounts_ix = valid_ix.clone(); empty_proof_accounts_ix .accounts - .truncate(empty_proof_accounts_ix.accounts.len() - proof_depth); + .truncate(empty_proof_accounts_ix.accounts.len() - entry.proof.len()); let result = rpc .create_and_send_transaction(&[empty_proof_accounts_ix], &forester.pubkey(), &[&forester]) .await; assert_rpc_error(result, 0, RegistryError::InvalidProofAccountsLength.into()).unwrap(); - let malformed_ix = Instruction { - program_id: light_registry::ID, - accounts: valid_ix.accounts.clone(), - data: light_registry::instruction::Nullify2 { - bump: 255, - change_log_indices: vec![change_log_index, change_log_index + 1], - leaves_queue_indices: vec![queue_index], - indices: vec![leaf_index], - } - .data(), - }; - let result = rpc - .create_and_send_transaction(&[malformed_ix], &forester.pubkey(), &[&forester]) - .await; - assert_rpc_error(result, 0, RegistryError::InvalidNullify2Inputs.into()).unwrap(); - + // Test: success with valid instruction. rpc.create_and_send_transaction(&[valid_ix], &forester.pubkey(), &[&forester]) .await .unwrap(); @@ -146,87 +170,19 @@ async fn test_nullify_2_validation_and_success() { #[serial] #[tokio::test] async fn test_legacy_nullify_still_succeeds() { - let mut rpc = LightProgramTest::new(ProgramTestConfig::default_with_batched_trees(true)) - .await - .unwrap(); - rpc.indexer = None; - let env = rpc.test_accounts.clone(); - let forester = Keypair::new(); - rpc.airdrop_lamports(&forester.pubkey(), 1_000_000_000) - .await - .unwrap(); - let merkle_tree_keypair = Keypair::new(); - let nullifier_queue_keypair = Keypair::new(); - let cpi_context_keypair = Keypair::new(); - let (mut rpc, state_tree_bundle) = { - let mut e2e_env = init_program_test_env(rpc, &env, 50).await; - e2e_env.indexer.state_merkle_trees.clear(); - e2e_env.keypair_action_config.fee_assert = false; - e2e_env - .indexer - .add_state_merkle_tree( - &mut e2e_env.rpc, - &merkle_tree_keypair, - &nullifier_queue_keypair, - &cpi_context_keypair, - None, - Some(forester.pubkey()), - TreeType::StateV1, - ) - .await; - e2e_env - .compress_sol_deterministic(&forester, 1_000_000, None) - .await; - e2e_env - .transfer_sol_deterministic(&forester, &Pubkey::new_unique(), None) - .await - .unwrap(); - (e2e_env.rpc, e2e_env.indexer.state_merkle_trees[0].clone()) - }; - let nullifier_queue = unsafe { - get_hash_set::(&mut rpc, state_tree_bundle.accounts.nullifier_queue).await - } - .unwrap(); - let mut queue_index = None; - let mut account_hash = None; - for i in 0..nullifier_queue.get_capacity() { - let bucket = nullifier_queue.get_bucket(i).unwrap(); - if let Some(bucket) = bucket { - if bucket.sequence_number.is_none() { - queue_index = Some(i as u16); - account_hash = Some(bucket.value_bytes()); - break; - } - } - } - let queue_index = queue_index.unwrap(); - let account_hash = account_hash.unwrap(); - let leaf_index = state_tree_bundle - .merkle_tree - .get_leaf_index(&account_hash) - .unwrap() as u64; - let proof = state_tree_bundle - .merkle_tree - .get_proof_of_leaf(leaf_index as usize, false) - .unwrap(); - let onchain_tree = - get_concurrent_merkle_tree::( - &mut rpc, - state_tree_bundle.accounts.merkle_tree, - ) - .await - .unwrap(); - let change_log_index = onchain_tree.changelog_index() as u64; + let (mut rpc, state_tree_bundle, forester) = setup_tree_with_nullifier_queue_entries(1).await; + let entries = read_queue_entries(&mut rpc, &state_tree_bundle, 1).await; + let entry = &entries[0]; let legacy_ix = create_nullify_instruction( CreateNullifyInstructionInputs { authority: forester.pubkey(), nullifier_queue: state_tree_bundle.accounts.nullifier_queue, merkle_tree: state_tree_bundle.accounts.merkle_tree, - change_log_indices: vec![change_log_index], - leaves_queue_indices: vec![queue_index], - indices: vec![leaf_index], - proofs: vec![proof], + change_log_indices: vec![entry.change_log_index], + leaves_queue_indices: vec![entry.queue_index], + indices: vec![entry.leaf_index], + proofs: vec![entry.proof.clone()], derivation: forester.pubkey(), is_metadata_forester: true, }, @@ -236,3 +192,49 @@ async fn test_legacy_nullify_still_succeeds() { .await .unwrap(); } + +#[serial] +#[tokio::test] +async fn test_paired_nullify_2_in_single_transaction() { + let (mut rpc, state_tree_bundle, forester) = setup_tree_with_nullifier_queue_entries(2).await; + let entries = read_queue_entries(&mut rpc, &state_tree_bundle, 2).await; + assert!( + entries.len() >= 2, + "need at least 2 queue entries, got {}", + entries.len() + ); + + let ix_0 = create_nullify_2_instruction( + CreateNullify2InstructionInputs { + authority: forester.pubkey(), + nullifier_queue: state_tree_bundle.accounts.nullifier_queue, + merkle_tree: state_tree_bundle.accounts.merkle_tree, + change_log_index: entries[0].change_log_index, + leaves_queue_index: entries[0].queue_index, + index: entries[0].leaf_index, + proof: entries[0].proof.clone().try_into().unwrap(), + derivation: forester.pubkey(), + is_metadata_forester: true, + }, + 0, + ); + let ix_1 = create_nullify_2_instruction( + CreateNullify2InstructionInputs { + authority: forester.pubkey(), + nullifier_queue: state_tree_bundle.accounts.nullifier_queue, + merkle_tree: state_tree_bundle.accounts.merkle_tree, + change_log_index: entries[1].change_log_index, + leaves_queue_index: entries[1].queue_index, + index: entries[1].leaf_index, + proof: entries[1].proof.clone().try_into().unwrap(), + derivation: forester.pubkey(), + is_metadata_forester: true, + }, + 0, + ); + + // Both nullify_2 instructions in a single transaction (the core pairing use case). + rpc.create_and_send_transaction(&[ix_0, ix_1], &forester.pubkey(), &[&forester]) + .await + .unwrap(); +} diff --git a/programs/registry/src/account_compression_cpi/nullify.rs b/programs/registry/src/account_compression_cpi/nullify.rs index c936f6e216..a4f03d8511 100644 --- a/programs/registry/src/account_compression_cpi/nullify.rs +++ b/programs/registry/src/account_compression_cpi/nullify.rs @@ -3,9 +3,9 @@ use account_compression::{ }; use anchor_lang::prelude::*; -use crate::{epoch::register_epoch::ForesterEpochPda, errors::RegistryError}; +use crate::epoch::register_epoch::ForesterEpochPda; -const NULLIFY_2_PROOF_ACCOUNTS_LEN: usize = 16; +pub(crate) const NULLIFY_2_PROOF_ACCOUNTS_LEN: usize = 16; #[derive(Accounts)] pub struct NullifyLeaves<'info> { @@ -72,57 +72,3 @@ pub(crate) fn extract_proof_nodes_from_remaining_accounts( .map(|account_info| account_info.key().to_bytes()) .collect() } - -pub(crate) fn validate_nullify_2_inputs( - change_log_indices: &[u64], - leaves_queue_indices: &[u16], - indices: &[u64], - proof_accounts_len: usize, -) -> Result<()> { - if change_log_indices.len() != 1 || leaves_queue_indices.len() != 1 || indices.len() != 1 { - return err!(RegistryError::InvalidNullify2Inputs); - } - if proof_accounts_len != NULLIFY_2_PROOF_ACCOUNTS_LEN { - return err!(RegistryError::InvalidProofAccountsLength); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::validate_nullify_2_inputs; - use crate::errors::RegistryError; - - #[test] - fn nullify_2_inputs_validate_happy_path() { - let result = validate_nullify_2_inputs(&[1], &[1], &[42], 16); - assert!(result.is_ok()); - } - - #[test] - fn nullify_2_inputs_reject_empty_proof_accounts() { - let result = validate_nullify_2_inputs(&[1], &[1], &[42], 0); - assert_eq!( - result.err().unwrap(), - RegistryError::InvalidProofAccountsLength.into() - ); - } - - #[test] - fn nullify_2_inputs_reject_vector_length_mismatch() { - let result = validate_nullify_2_inputs(&[1, 2], &[1], &[42], 16); - assert_eq!( - result.err().unwrap(), - RegistryError::InvalidNullify2Inputs.into() - ); - } - - #[test] - fn nullify_2_inputs_reject_invalid_proof_accounts_length() { - let result = validate_nullify_2_inputs(&[1], &[1], &[42], 15); - assert_eq!( - result.err().unwrap(), - RegistryError::InvalidProofAccountsLength.into() - ); - } -} diff --git a/programs/registry/src/account_compression_cpi/sdk.rs b/programs/registry/src/account_compression_cpi/sdk.rs index 0e32f17282..7ee3e1a575 100644 --- a/programs/registry/src/account_compression_cpi/sdk.rs +++ b/programs/registry/src/account_compression_cpi/sdk.rs @@ -87,9 +87,9 @@ pub fn create_nullify_2_instruction( let (cpi_authority, bump) = get_cpi_authority_pda(); let instruction_data = crate::instruction::Nullify2 { bump, - change_log_indices: vec![inputs.change_log_index], - leaves_queue_indices: vec![inputs.leaves_queue_index], - indices: vec![inputs.index], + change_log_index: inputs.change_log_index, + leaves_queue_index: inputs.leaves_queue_index, + index: inputs.index, }; let base_accounts = crate::accounts::NullifyLeaves { @@ -678,7 +678,7 @@ mod tests { } assert_eq!(&ix.data[..8], crate::instruction::Nullify2::DISCRIMINATOR); - // 8-byte discriminator + 31-byte minimal payload. - assert_eq!(ix.data.len(), 39); + // 8-byte discriminator + 19-byte scalar payload (u8 + u64 + u16 + u64). + assert_eq!(ix.data.len(), 27); } } diff --git a/programs/registry/src/errors.rs b/programs/registry/src/errors.rs index 73384abb41..e0a176fb41 100644 --- a/programs/registry/src/errors.rs +++ b/programs/registry/src/errors.rs @@ -38,12 +38,10 @@ pub enum RegistryError { InvalidTokenAccountData, #[msg("Indices array cannot be empty")] EmptyIndices, - #[msg("Nullify2 proof accounts length is invalid")] - InvalidProofAccountsLength, - #[msg("Nullify2 supports exactly one change, queue index, and leaf index")] - InvalidNullify2Inputs, #[msg("Failed to borrow account data")] BorrowAccountDataFailed, #[msg("Failed to serialize instruction data")] SerializationFailed, + #[msg("Nullify2 proof accounts length is invalid")] + InvalidProofAccountsLength, } diff --git a/programs/registry/src/lib.rs b/programs/registry/src/lib.rs index 16a5d56548..d4ce2f2aa0 100644 --- a/programs/registry/src/lib.rs +++ b/programs/registry/src/lib.rs @@ -12,7 +12,7 @@ use light_merkle_tree_metadata::merkle_tree::MerkleTreeMetadata; pub mod account_compression_cpi; pub mod errors; use account_compression_cpi::nullify::{ - extract_proof_nodes_from_remaining_accounts, validate_nullify_2_inputs, + extract_proof_nodes_from_remaining_accounts, NULLIFY_2_PROOF_ACCOUNTS_LEN, }; pub use account_compression_cpi::{ batch_append::*, batch_nullify::*, batch_update_address_tree::*, @@ -426,9 +426,9 @@ pub mod light_registry { pub fn nullify_2<'info>( ctx: Context<'_, '_, '_, 'info, NullifyLeaves<'info>>, bump: u8, - change_log_indices: Vec, - leaves_queue_indices: Vec, - indices: Vec, + change_log_index: u64, + leaves_queue_index: u16, + index: u64, ) -> Result<()> { let metadata = ctx.accounts.merkle_tree.load()?.metadata; check_forester( @@ -439,20 +439,17 @@ pub mod light_registry { DEFAULT_WORK_V1, )?; - validate_nullify_2_inputs( - &change_log_indices, - &leaves_queue_indices, - &indices, - ctx.remaining_accounts.len(), - )?; + if ctx.remaining_accounts.len() != NULLIFY_2_PROOF_ACCOUNTS_LEN { + return err!(RegistryError::InvalidProofAccountsLength); + } let proof_nodes = extract_proof_nodes_from_remaining_accounts(ctx.remaining_accounts); process_nullify( &ctx, bump, - change_log_indices, - leaves_queue_indices, - indices, + vec![change_log_index], + vec![leaves_queue_index], + vec![index], vec![proof_nodes], ) }