diff --git a/common/nym-lp/src/codec.rs b/common/nym-lp/src/codec.rs index 29af2fc788c..0d549951d17 100644 --- a/common/nym-lp/src/codec.rs +++ b/common/nym-lp/src/codec.rs @@ -16,8 +16,8 @@ pub const OUTER_HEADER_SIZE: usize = OuterHeader::SIZE; // 12 bytes /// Size of inner prefix (proto + reserved) - cleartext or encrypted depending on mode const INNER_PREFIX_SIZE: usize = 4; // proto(1) + reserved(3) use chacha20poly1305::{ - aead::{AeadInPlace, KeyInit}, ChaCha20Poly1305, Key, Nonce, Tag, + aead::{AeadInPlace, KeyInit}, }; use zeroize::{Zeroize, ZeroizeOnDrop}; @@ -32,7 +32,7 @@ use zeroize::{Zeroize, ZeroizeOnDrop}; /// ChaCha20-Poly1305 requires unique nonces per key. The counter starts at 0 /// for each session, which is safe because: /// -/// 1. **PSK is always fresh**: Each handshake uses PSQ +/// 1. **PSK is always fresh**: Each handshake uses PSQ /// with a client-generated random salt. This ensures a unique /// PSK for every session, even between the same client-gateway pair. /// @@ -106,9 +106,9 @@ fn parse_message_from_type_and_content( Ok(LpMessage::Busy) } MessageType::Handshake => Ok(LpMessage::Handshake(HandshakeData(content.to_vec()))), - MessageType::EncryptedData => { - Ok(LpMessage::EncryptedData(EncryptedDataPayload(content.to_vec()))) - } + MessageType::EncryptedData => Ok(LpMessage::EncryptedData(EncryptedDataPayload( + content.to_vec(), + ))), MessageType::ClientHello => { let data: ClientHelloData = bincode::deserialize(content) .map_err(|e| LpError::DeserializationError(e.to_string()))?; @@ -205,10 +205,7 @@ pub fn parse_lp_header_only(src: &[u8]) -> Result { /// # Errors /// * `LpError::AeadTagMismatch` - Tag verification failed (when outer_key provided) /// * `LpError::InsufficientBufferSize` - Packet too small -pub fn parse_lp_packet( - src: &[u8], - outer_key: Option<&OuterAeadKey>, -) -> Result { +pub fn parse_lp_packet(src: &[u8], outer_key: Option<&OuterAeadKey>) -> Result { // Minimum size check: OuterHeader + InnerPrefix + MsgType + Trailer (for 0-payload message) // 12 + 4 + 2 + 16 = 34 bytes let min_size = OUTER_HEADER_SIZE + INNER_PREFIX_SIZE + 2 + TRAILER_LEN; @@ -391,7 +388,7 @@ impl LpError { #[cfg(test)] mod tests { // Import standalone functions - use super::{parse_lp_packet, serialize_lp_packet, OuterAeadKey}; + use super::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; // Keep necessary imports use crate::LpError; use crate::message::{EncryptedDataPayload, HandshakeData, LpMessage, MessageType}; @@ -1306,7 +1303,10 @@ mod tests { let result = parse_lp_packet(&buf, None); assert!(matches!( result, - Err(LpError::InvalidPayloadSize { expected: 0, actual: 1 }) + Err(LpError::InvalidPayloadSize { + expected: 0, + actual: 1 + }) )); } diff --git a/common/nym-lp/src/lib.rs b/common/nym-lp/src/lib.rs index 89bd4e72f44..c761892080a 100644 --- a/common/nym-lp/src/lib.rs +++ b/common/nym-lp/src/lib.rs @@ -16,7 +16,7 @@ pub mod session_manager; pub use error::LpError; pub use message::{ClientHelloData, LpMessage}; -pub use packet::{LpPacket, OuterHeader, BOOTSTRAP_RECEIVER_IDX}; +pub use packet::{BOOTSTRAP_RECEIVER_IDX, LpPacket, OuterHeader}; pub use replay::{ReceivingKeyCounterValidator, ReplayError}; pub use session::{LpSession, generate_fresh_salt}; pub use session_manager::SessionManager; @@ -315,8 +315,8 @@ mod tests { let parsed_packet3 = parse_lp_packet(&buf3, None).unwrap(); // Perform replay check (should fail) - let replay_result = - local_manager.receiving_counter_quick_check(receiver_index, parsed_packet3.header.counter); + let replay_result = local_manager + .receiving_counter_quick_check(receiver_index, parsed_packet3.header.counter); assert!(replay_result.is_err()); match replay_result.unwrap_err() { LpError::Replay(e) => { diff --git a/common/nym-lp/src/message.rs b/common/nym-lp/src/message.rs index 006a8ca22eb..6c0c7b1427b 100644 --- a/common/nym-lp/src/message.rs +++ b/common/nym-lp/src/message.rs @@ -233,7 +233,7 @@ impl LpMessage { LpMessage::SubsessionKK1(_) => false, // Always has payload LpMessage::SubsessionKK2(_) => false, // Always has payload LpMessage::SubsessionReady(_) => false, // Always has receiver_index - LpMessage::SubsessionAbort => true, // Empty signal + LpMessage::SubsessionAbort => true, // Empty signal } } diff --git a/common/nym-lp/src/session.rs b/common/nym-lp/src/session.rs index 316849b211f..6ef3f91372e 100644 --- a/common/nym-lp/src/session.rs +++ b/common/nym-lp/src/session.rs @@ -11,7 +11,9 @@ use crate::keypair::{PrivateKey, PublicKey}; use crate::message::{EncryptedDataPayload, HandshakeData}; use crate::noise_protocol::{NoiseError, NoiseProtocol, ReadResult}; use crate::packet::LpHeader; -use crate::psk::{derive_subsession_psk, psq_initiator_create_message, psq_responder_process_message}; +use crate::psk::{ + derive_subsession_psk, psq_initiator_create_message, psq_responder_process_message, +}; use crate::replay::ReceivingKeyCounterValidator; use crate::{LpError, LpMessage, LpPacket}; use nym_crypto::asymmetric::ed25519; @@ -117,8 +119,12 @@ pub enum PSQState { NotStarted, /// Initiator has sent PSQ ciphertext and is waiting for confirmation. - /// Stores the ciphertext that was sent. - InitiatorWaiting { ciphertext: Vec }, + /// PSK is already derived but we don't encrypt outgoing packets yet + /// because the responder may not have processed our message yet. + InitiatorWaiting { + /// The derived PSK, stored until we transition to Completed + psk: [u8; 32], + }, /// Responder is ready to receive and decapsulate PSQ ciphertext. ResponderWaiting, @@ -280,10 +286,40 @@ impl LpSession { /// /// Callers should use `None` for packet encryption/decryption during /// the handshake phase, and use the returned key for transport phase. + /// + /// Note: For sending packets during handshake, use `outer_aead_key_for_sending()` + /// which checks PSQ state to avoid encrypting before the responder can decrypt. pub fn outer_aead_key(&self) -> Option { self.outer_aead_key.lock().clone() } + /// Returns the outer AEAD key only if it's safe to use for sending. + /// + /// This method gates the key based on PSQ handshake state: + /// - Returns `None` if PSQ is NotStarted, InitiatorWaiting, or ResponderWaiting + /// - Returns `Some(key)` only if PSQ is Completed + /// + /// # Why This Matters + /// + /// The first Noise handshake message (containing PSQ payload from initiator) + /// must be sent in cleartext because the responder hasn't derived the PSK yet. + /// Only after the responder processes the PSQ and both sides have the PSK + /// can outer encryption be used for sending. + /// + /// For receiving, use `outer_aead_key()` which returns the key as soon as + /// it's derived (needed because the peer may start encrypting before we've + /// finished our send). + // This fixes a bug where the initiator encrypted the first Noise + // message with outer AEAD, but the responder couldn't decrypt because it + // hadn't processed the PSQ yet to derive the same PSK. + pub fn outer_aead_key_for_sending(&self) -> Option { + let psq_state = self.psq_state.lock(); + match &*psq_state { + PSQState::Completed { .. } => self.outer_aead_key.lock().clone(), + _ => None, + } + } + /// Creates a new session and initializes the Noise protocol state. /// /// PSQ always runs during the handshake to derive the real PSK from X25519 DHKEM. @@ -735,10 +771,9 @@ impl LpSession { combined.extend_from_slice(&psq_payload); combined.extend_from_slice(&noise_msg); - // Update PSQ state to InitiatorWaiting - *psq_state = PSQState::InitiatorWaiting { - ciphertext: psq_payload, - }; + // PSK is derived but we stay in InitiatorWaiting until we receive msg 2. + // This ensures we send msg 1 in cleartext (responder can't decrypt yet). + *psq_state = PSQState::InitiatorWaiting { psk }; return Some(Ok(LpMessage::Handshake(HandshakeData(combined)))); } @@ -875,7 +910,8 @@ impl LpSession { let psk = psq_result.psk; // Store PQ shared secret for subsession PSK derivation - *self.pq_shared_secret.lock() = Some(PqSharedSecret::new(psq_result.pq_shared_secret)); + *self.pq_shared_secret.lock() = + Some(PqSharedSecret::new(psq_result.pq_shared_secret)); // Store the PSK handle (ctxt_B) for transmission in next message { @@ -905,7 +941,9 @@ impl LpSession { } // Check if initiator should extract PSK handle from message 2 - if self.is_initiator && matches!(*psq_state, PSQState::InitiatorWaiting { .. }) { + if let PSQState::InitiatorWaiting { psk } = *psq_state + && self.is_initiator + { // Extract PSK handle: [u16 handle_len][handle_bytes][noise_msg] if payload.len() >= 2 { let handle_len = u16::from_le_bytes([payload[0], payload[1]]) as usize; @@ -925,7 +963,8 @@ impl LpSession { *psk_handle = Some(handle_bytes.to_vec()); } - // Release psq_state lock before processing + // Transition to Completed - we've received confirmation from responder + *psq_state = PSQState::Completed { psk }; drop(psq_state); // Process only the Noise message part @@ -1096,9 +1135,7 @@ impl LpSession { ) -> Result { // Verify parent handshake is complete if !self.is_handshake_complete() { - return Err(LpError::Internal( - "Parent handshake not complete".into(), - )); + return Err(LpError::Internal("Parent handshake not complete".into())); } // Get PQ shared secret @@ -1136,11 +1173,16 @@ impl LpSession { // Copy key material from parent for into_session() conversion local_ed25519_private: ed25519::PrivateKey::from_bytes( &self.local_ed25519_private.to_bytes(), - ).expect("Valid Ed25519 private key from parent"), - local_ed25519_public: ed25519::PublicKey::from_bytes(&self.local_ed25519_public.to_bytes()) - .expect("Valid Ed25519 public key from parent"), - remote_ed25519_public: ed25519::PublicKey::from_bytes(&self.remote_ed25519_public.to_bytes()) - .expect("Valid Ed25519 public key from parent"), + ) + .expect("Valid Ed25519 private key from parent"), + local_ed25519_public: ed25519::PublicKey::from_bytes( + &self.local_ed25519_public.to_bytes(), + ) + .expect("Valid Ed25519 public key from parent"), + remote_ed25519_public: ed25519::PublicKey::from_bytes( + &self.remote_ed25519_public.to_bytes(), + ) + .expect("Valid Ed25519 public key from parent"), local_x25519_private: self.local_x25519_private.clone(), remote_x25519_public: self.remote_x25519_public.clone(), pq_shared_secret: PqSharedSecret::new(pq_secret), @@ -1269,7 +1311,9 @@ impl SubsessionHandshake { // KKT: subsession inherits from parent, mark as processed kkt_state: Mutex::new(KKTState::ResponderProcessed), // PSQ: subsession uses PSK derived from parent's PQ secret - psq_state: Mutex::new(PSQState::Completed { psk: self.subsession_psk }), + psq_state: Mutex::new(PSQState::Completed { + psk: self.subsession_psk, + }), psk_handle: Mutex::new(None), // Subsession doesn't have its own handle sending_counter: AtomicU64::new(0), receiving_counter: Mutex::new(ReceivingKeyCounterValidator::new(0)), @@ -1435,8 +1479,12 @@ mod tests { let responder_keys = generate_keypair(); let receiver_index = 12345u32; - let initiator_session = - create_handshake_test_session(receiver_index, true, &initiator_keys, responder_keys.public_key()); + let initiator_session = create_handshake_test_session( + receiver_index, + true, + &initiator_keys, + responder_keys.public_key(), + ); let responder_session = create_handshake_test_session( receiver_index, false, @@ -1463,10 +1511,18 @@ mod tests { let responder_keys = generate_keypair(); let receiver_index = 12345u32; - let initiator_session = - create_handshake_test_session(receiver_index, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(receiver_index, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + receiver_index, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + receiver_index, + false, + &responder_keys, + initiator_keys.public_key(), + ); // 1. Initiator prepares the first message (-> e) let initiator_msg_result = initiator_session.prepare_handshake_message(); @@ -1500,10 +1556,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); let mut responder_to_initiator_msg = None; let mut rounds = 0; @@ -1587,10 +1651,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Drive handshake to completion (simplified loop from previous test) let mut i_msg = initiator_session @@ -1648,8 +1720,12 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); assert!(!initiator_session.is_handshake_complete()); @@ -1720,10 +1796,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Drive the handshake let mut i_msg = initiator_session @@ -1814,10 +1898,18 @@ mod tests { let responder_keys = generate_keypair(); // Create sessions - they start with dummy PSK [0u8; 32] - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Prepare first message (initiator runs PSQ and injects PSK) let i_msg = initiator_session @@ -1879,10 +1971,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Verify initial state assert!(!initiator_session.is_handshake_complete()); @@ -1958,10 +2058,18 @@ mod tests { let responder_keys = generate_keypair(); // Create sessions with explicit Ed25519 keys - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Verify sessions store Ed25519 keys // (Internal verification - keys are used in PSQ calls) @@ -2003,8 +2111,12 @@ mod tests { let responder_keys = generate_keypair(); let initiator_keys = generate_keypair(); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Create a handshake message with corrupted PSQ payload let corrupted_psq_data = vec![0xFF; 128]; // Random garbage @@ -2168,8 +2280,12 @@ mod tests { let responder_keys = generate_keypair(); let initiator_keys = generate_keypair(); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Capture initial PSQ state (should be ResponderWaiting) // (We can't directly access psq_state, but we can verify behavior) @@ -2186,8 +2302,12 @@ mod tests { // Session should still be functional - can process valid messages // Create a proper initiator to send valid message - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); let valid_msg = initiator_session .prepare_handshake_message() @@ -2213,8 +2333,12 @@ mod tests { let responder_keys = generate_keypair(); // Create session but don't complete handshake (no PSK injection will occur) - let session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); + let session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); // Verify session was created successfully assert!(!session.is_handshake_complete()); @@ -2257,8 +2381,12 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); + let session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); // Initially not read-only assert!(!session.is_read_only()); @@ -2278,10 +2406,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Drive handshake to completion let i_msg = initiator_session @@ -2326,10 +2462,18 @@ mod tests { let initiator_keys = generate_keypair(); let responder_keys = generate_keypair(); - let initiator_session = - create_handshake_test_session(12345u32, true, &initiator_keys, responder_keys.public_key()); - let responder_session = - create_handshake_test_session(12345u32, false, &responder_keys, initiator_keys.public_key()); + let initiator_session = create_handshake_test_session( + 12345u32, + true, + &initiator_keys, + responder_keys.public_key(), + ); + let responder_session = create_handshake_test_session( + 12345u32, + false, + &responder_keys, + initiator_keys.public_key(), + ); // Drive handshake to completion let i_msg = initiator_session diff --git a/common/nym-lp/src/session_integration/mod.rs b/common/nym-lp/src/session_integration/mod.rs index 0f5376639d5..ceafc3f81f2 100644 --- a/common/nym-lp/src/session_integration/mod.rs +++ b/common/nym-lp/src/session_integration/mod.rs @@ -149,7 +149,8 @@ mod tests { let counter = session_manager_1.next_counter(receiver_index).unwrap(); let message_a_to_b = create_test_packet(1, receiver_index, counter, payload); let mut encoded_msg = BytesMut::new(); - serialize_lp_packet(&message_a_to_b, &mut encoded_msg, None).expect("A serialize failed"); + serialize_lp_packet(&message_a_to_b, &mut encoded_msg, None) + .expect("A serialize failed"); // B parses packet and checks replay let decoded_packet = parse_lp_packet(&encoded_msg, None).expect("B parse failed"); @@ -200,7 +201,8 @@ mod tests { let counter = session_manager_2.next_counter(peer_b_sm).unwrap(); let message_b_to_a = create_test_packet(1, receiver_index, counter, payload); let mut encoded_msg = BytesMut::new(); - serialize_lp_packet(&message_b_to_a, &mut encoded_msg, None).expect("B serialize failed"); + serialize_lp_packet(&message_b_to_a, &mut encoded_msg, None) + .expect("B serialize failed"); // A parses packet and checks replay let decoded_packet = parse_lp_packet(&encoded_msg, None).expect("A parse failed"); @@ -289,7 +291,8 @@ mod tests { .expect("A serialize data failed"); // B parses packet and checks replay - let decoded_packet_b = parse_lp_packet(&encoded_data_a_to_b, None).expect("B parse data failed"); + let decoded_packet_b = + parse_lp_packet(&encoded_data_a_to_b, None).expect("B parse data failed"); assert_eq!(decoded_packet_b.header.counter, counter_a); // Check replay before decrypting @@ -323,7 +326,8 @@ mod tests { .expect("B serialize data failed"); // A parses packet and checks replay - let decoded_packet_a = parse_lp_packet(&encoded_data_b_to_a, None).expect("A parse data failed"); + let decoded_packet_a = + parse_lp_packet(&encoded_data_b_to_a, None).expect("A parse data failed"); assert_eq!(decoded_packet_a.header.counter, counter_b); // Check replay before decrypting @@ -360,8 +364,12 @@ mod tests { )), // Using plaintext here, but content doesn't matter for replay check ); let mut encoded_data_b_to_a_replay = BytesMut::new(); - serialize_lp_packet(&message_b_to_a_replay, &mut encoded_data_b_to_a_replay, None) - .expect("B serialize replay failed"); + serialize_lp_packet( + &message_b_to_a_replay, + &mut encoded_data_b_to_a_replay, + None, + ) + .expect("B serialize replay failed"); let parsed_replay_packet = parse_lp_packet(&encoded_data_b_to_a_replay, None).expect("A parse replay failed"); @@ -398,7 +406,8 @@ mod tests { .expect("Failed to serialize skip message"); // B parses skip message and checks replay - let decoded_packet_skip = parse_lp_packet(&encoded_skip, None).expect("B parse skip failed"); + let decoded_packet_skip = + parse_lp_packet(&encoded_skip, None).expect("B parse skip failed"); session_manager_2 .receiving_counter_quick_check(peer_b_sm, decoded_packet_skip.header.counter) .expect("B replay check skip failed"); @@ -840,22 +849,6 @@ mod tests { let ed25519_keypair_a = ed25519::KeyPair::from_secret([6u8; 32], 0); let ed25519_keypair_b = ed25519::KeyPair::from_secret([7u8; 32], 1); - // Derive X25519 keys from Ed25519 (same as state machine does internally) - let x25519_pub_a = ed25519_keypair_a - .public_key() - .to_x25519() - .expect("Failed to derive X25519 from Ed25519"); - let x25519_pub_b = ed25519_keypair_b - .public_key() - .to_x25519() - .expect("Failed to derive X25519 from Ed25519"); - - // Convert to LP keypair types (needed for init_kkt_for_test if used) - let lp_pub_a = PublicKey::from_bytes(x25519_pub_a.as_bytes()) - .expect("Failed to create PublicKey from bytes"); - let lp_pub_b = PublicKey::from_bytes(x25519_pub_b.as_bytes()) - .expect("Failed to create PublicKey from bytes"); - // Use fixed receiver_index for test let receiver_index: u32 = 100005; @@ -940,7 +933,8 @@ mod tests { " Round {}: Responder explicitly enters KKTExchange state", rounds ); - let action_b_start = session_manager_2.process_input(receiver_index, LpInput::StartHandshake); + let action_b_start = + session_manager_2.process_input(receiver_index, LpInput::StartHandshake); // Responder's StartHandshake should not produce an action to send assert!( action_b_start.as_ref().unwrap().is_none(), @@ -1030,7 +1024,9 @@ mod tests { // KKT completed, now need to explicitly trigger handshake message // This might be the case if KKT completion doesn't automatically send the first Noise message // Let's try to prepare the handshake message - if let Some(msg_result) = session_manager_1.prepare_handshake_message(receiver_index) { + if let Some(msg_result) = + session_manager_1.prepare_handshake_message(receiver_index) + { let msg = msg_result.expect("Failed to prepare handshake message after KKT"); // Create a packet from the message let packet = create_test_packet(1, receiver_index, 0, msg); @@ -1252,8 +1248,8 @@ mod tests { // --- 6. Replay Protection Test --- println!("Testing data packet replay protection via process_input..."); - let replay_result = - session_manager_1.process_input(receiver_index, LpInput::ReceivePacket(data_packet_b_replay)); // Use cloned packet + let replay_result = session_manager_1 + .process_input(receiver_index, LpInput::ReceivePacket(data_packet_b_replay)); // Use cloned packet assert!(replay_result.is_err(), "Replay should produce Err(...)"); let error = replay_result.err().unwrap(); @@ -1314,8 +1310,8 @@ mod tests { // B tries to replay N (should fail) println!(" B tries to replay N"); - let replay_n_result = - session_manager_2.process_input(receiver_index, LpInput::ReceivePacket(packet_n_replay)); + let replay_n_result = session_manager_2 + .process_input(receiver_index, LpInput::ReceivePacket(packet_n_replay)); assert!(replay_n_result.is_err(), "Replay N should produce Err"); assert!( matches!(replay_n_result.err().unwrap(), LpError::Replay(_)), diff --git a/common/nym-lp/src/session_manager.rs b/common/nym-lp/src/session_manager.rs index 423ba749240..8eb1bbcb3d9 100644 --- a/common/nym-lp/src/session_manager.rs +++ b/common/nym-lp/src/session_manager.rs @@ -42,7 +42,9 @@ impl SessionManager { pub fn add(&self, session: LpSession) -> Result<(), LpError> { let sm = LpStateMachine { - state: LpState::ReadyToHandshake { session }, + state: LpState::ReadyToHandshake { + session: Box::new(session), + }, }; self.state_machines.insert(sm.id()?, sm); Ok(()) diff --git a/common/nym-lp/src/state_machine.rs b/common/nym-lp/src/state_machine.rs index 8fd3b5d741e..984cc3011ac 100644 --- a/common/nym-lp/src/state_machine.rs +++ b/common/nym-lp/src/state_machine.rs @@ -21,29 +21,29 @@ use tracing::debug; pub enum LpState { /// Initial state: Ready to start the handshake. /// State machine is created with keys, lp_id is derived, session is ready. - ReadyToHandshake { session: LpSession }, + ReadyToHandshake { session: Box }, /// Performing KKT (KEM Key Transfer) exchange before Noise handshake. /// Initiator requests responder's KEM public key, responder provides signed key. - KKTExchange { session: LpSession }, + KKTExchange { session: Box }, /// Actively performing the Noise handshake. /// (We might be able to merge this with ReadyToHandshake if the first step always happens) - Handshaking { session: LpSession }, // Kept for now, logic might merge later + Handshaking { session: Box }, // Kept for now, logic might merge later /// Handshake complete, ready for data transport. - Transport { session: LpSession }, + Transport { session: Box }, /// Performing subsession KK handshake while parent remains active. /// Parent can still send/receive; subsession messages tunneled through parent. SubsessionHandshaking { - session: LpSession, - subsession: SubsessionHandshake, + session: Box, + subsession: Box, }, /// Parent session demoted after subsession promoted. /// Can only receive (drain in-flight), cannot send. - ReadOnlyTransport { session: LpSession }, + ReadOnlyTransport { session: Box }, /// An error occurred, or the connection was intentionally closed. Closed { reason: String }, @@ -119,7 +119,7 @@ pub enum LpAction { /// the completed SubsessionHandshake for into_session(), and the new receiver_index. SubsessionComplete { packet: Option, - subsession: SubsessionHandshake, + subsession: Box, new_receiver_index: u32, }, } @@ -157,7 +157,7 @@ impl LpStateMachine { | LpState::Handshaking { session } | LpState::Transport { session } | LpState::SubsessionHandshaking { session, .. } - | LpState::ReadOnlyTransport { session } => Ok(session), + | LpState::ReadOnlyTransport { session } => Ok(*session), LpState::Closed { .. } => Err(LpError::LpSessionClosed), LpState::Processing => Err(LpError::LpSessionProcessing), } @@ -234,7 +234,9 @@ impl LpStateMachine { )?; Ok(LpStateMachine { - state: LpState::ReadyToHandshake { session }, + state: LpState::ReadyToHandshake { + session: Box::new(session), + }, }) } @@ -257,7 +259,9 @@ impl LpStateMachine { ) -> Result { let session = subsession.into_session(receiver_index)?; Ok(LpStateMachine { - state: LpState::Transport { session }, + state: LpState::Transport { + session: Box::new(session), + }, }) } @@ -538,7 +542,7 @@ impl LpStateMachine { Ok(response_packet) => { result_action = Some(Ok(LpAction::SendPacket(response_packet))); // Stay in SubsessionHandshaking, wait for SubsessionReady - LpState::SubsessionHandshaking { session, subsession } + LpState::SubsessionHandshaking { session, subsession: Box::new(subsession) } } Err(e) => { let reason = e.to_string(); @@ -596,7 +600,7 @@ impl LpStateMachine { } } } - // AIDEV-NOTE: Stale abort in Transport state - race already resolved. + // Stale abort in Transport state - race already resolved. // This can happen if abort arrives after loser already returned to Transport // via KK1 processing (loser detected local < remote and became responder). // The winner's abort message arrived late. Silently ignore. @@ -659,7 +663,7 @@ impl LpStateMachine { packet, subsession_index, })); - LpState::SubsessionHandshaking { session, subsession } + LpState::SubsessionHandshaking { session, subsession: Box::new(subsession) } } Err(e) => { let reason = e.to_string(); @@ -753,7 +757,7 @@ impl LpStateMachine { Ok(response_packet) => { result_action = Some(Ok(LpAction::SendPacket(response_packet))); // Replace old initiator subsession with new responder subsession - LpState::SubsessionHandshaking { session, subsession: new_subsession } + LpState::SubsessionHandshaking { session, subsession: Box::new(new_subsession) } } Err(e) => { let reason = e.to_string(); @@ -955,8 +959,7 @@ impl LpStateMachine { if packet.header.receiver_idx() != session.id() { result_action = Some(Err(LpError::UnknownSessionId(packet.header.receiver_idx()))); LpState::ReadOnlyTransport { session } - } else { - if let Err(e) = session.receiving_counter_quick_check(packet.header.counter) { + } else if let Err(e) = session.receiving_counter_quick_check(packet.header.counter) { result_action = Some(Err(e)); LpState::ReadOnlyTransport { session } } else { @@ -976,7 +979,6 @@ impl LpStateMachine { LpState::Closed { reason } } } - } } } @@ -1613,7 +1615,12 @@ mod tests { // --- Complete Noise Handshake --- // Alice prepares first Noise message - let noise1_msg = alice.session().unwrap().prepare_handshake_message().unwrap().unwrap(); + let noise1_msg = alice + .session() + .unwrap() + .prepare_handshake_message() + .unwrap() + .unwrap(); let noise1_packet = alice.session().unwrap().next_packet(noise1_msg).unwrap(); // Bob receives noise1, sends noise2 @@ -1666,10 +1673,7 @@ mod tests { } else { panic!("Alice should initiate subsession with KK1"); }; - assert!(matches!( - alice.state, - LpState::SubsessionHandshaking { .. } - )); + assert!(matches!(alice.state, LpState::SubsessionHandshaking { .. })); // Bob initiates subsession (simultaneously) let bob_kk1_packet = if let Some(Ok(LpAction::SubsessionInitiated { packet, .. })) = diff --git a/common/registration/src/lp_messages.rs b/common/registration/src/lp_messages.rs index 6c554450ccc..5f4d4e04923 100644 --- a/common/registration/src/lp_messages.rs +++ b/common/registration/src/lp_messages.rs @@ -144,11 +144,9 @@ mod tests { #[test] fn test_lp_registration_response_success() { let gateway_data = create_test_gateway_data(); - let session_id = 12345; let allocated_bandwidth = 1_000_000_000; - let response = - LpRegistrationResponse::success(allocated_bandwidth, gateway_data.clone()); + let response = LpRegistrationResponse::success(allocated_bandwidth, gateway_data.clone()); assert!(response.success); assert!(response.error.is_none()); diff --git a/docker/localnet/Dockerfile.localnet b/docker/localnet/Dockerfile.localnet index 9c1f6a30afb..a6eb260f1a9 100644 --- a/docker/localnet/Dockerfile.localnet +++ b/docker/localnet/Dockerfile.localnet @@ -26,6 +26,7 @@ RUN apt update && apt install -y \ wireguard-tools \ golang-go \ git \ + iptables \ && rm -rf /var/lib/apt/lists/* # Install wireguard-go (userspace WireGuard implementation) diff --git a/docker/localnet/README.md b/docker/localnet/README.md index 19a2f37a5e8..415b38e01b1 100644 --- a/docker/localnet/README.md +++ b/docker/localnet/README.md @@ -77,12 +77,12 @@ Host Machine (macOS) Ports published to host: - 1080 → SOCKS5 proxy -- 9000 → Gateway entry -- 10001-10004 → Mixnet ports -- 20001-20004 → Verloc ports -- 30001-30004 → HTTP APIs -- 41264 → LP control port (registration) -- 51264 → LP data port +- 9000/9001 → Gateway entry ports +- 10001-10005 → Mixnet ports +- 20001-20005 → Verloc ports +- 30001-30005 → HTTP APIs +- 41264/41265 → LP control ports (registration) +- 51822/51823 → WireGuard tunnel ports (gateway/gateway2) ### Startup Flow diff --git a/docker/localnet/localnet.sh b/docker/localnet/localnet.sh index 850df6258b1..e021310bc24 100755 --- a/docker/localnet/localnet.sh +++ b/docker/localnet/localnet.sh @@ -234,6 +234,7 @@ start_gateway() { -p 30004:30004 \ -p 41264:41264 \ -p 51264:51264 \ + -p 51822:51822/udp \ -v "$VOLUME_PATH:/localnet" \ -v "$NYM_VOLUME_PATH:/root/.nym" \ -d \ @@ -300,6 +301,7 @@ start_gateway2() { -p 30005:30005 \ -p 41265:41265 \ -p 51265:51265 \ + -p 51823:51822/udp \ -v "$VOLUME_PATH:/localnet" \ -v "$NYM_VOLUME_PATH:/root/.nym" \ -d \ @@ -606,6 +608,20 @@ start_all() { start_gateway start_gateway2 build_topology + + # Configure networking for two-hop WireGuard routing on both gateways + # Note: Runs after build_topology to ensure gateways have finished WireGuard setup + log_info "Configuring gateway networking (IP forwarding, NAT)..." + for gw in "$GATEWAY_CONTAINER" "$GATEWAY2_CONTAINER"; do + container exec "$gw" sh -c " + # Enable IP forwarding + echo 1 > /proc/sys/net/ipv4/ip_forward + # Add NAT masquerade for outbound traffic + iptables-legacy -t nat -A POSTROUTING -o eth0 -j MASQUERADE + " + log_success "Configured $gw" + done + start_network_requester start_socks5_client diff --git a/gateway/src/node/lp_listener/handler.rs b/gateway/src/node/lp_listener/handler.rs index caf3e1b0afd..e7a2d134ffd 100644 --- a/gateway/src/node/lp_listener/handler.rs +++ b/gateway/src/node/lp_listener/handler.rs @@ -6,8 +6,8 @@ use super::registration::process_registration; use super::LpHandlerState; use crate::error::GatewayError; use nym_lp::{ - codec::OuterAeadKey, keypair::PublicKey, message::ForwardPacketData, OuterHeader, - LpMessage, LpPacket, + codec::OuterAeadKey, keypair::PublicKey, message::ForwardPacketData, packet::LpHeader, + LpMessage, LpPacket, OuterHeader, }; use nym_metrics::{add_histogram_obs, inc}; use std::net::SocketAddr; @@ -135,11 +135,12 @@ impl LpConnectionHandler { }; // Step 3: Parse full packet with outer AEAD key - let packet = nym_lp::codec::parse_lp_packet(&raw_bytes, outer_key.as_ref()).map_err(|e| { - inc!("lp_errors_parse_packet"); - self.emit_lifecycle_metrics(false); - GatewayError::LpProtocolError(format!("Failed to parse LP packet: {}", e)) - })?; + let packet = + nym_lp::codec::parse_lp_packet(&raw_bytes, outer_key.as_ref()).map_err(|e| { + inc!("lp_errors_parse_packet"); + self.emit_lifecycle_metrics(false); + GatewayError::LpProtocolError(format!("Failed to parse LP packet: {}", e)) + })?; trace!( "Received packet from {} (receiver_idx={}, counter={}, encrypted={})", @@ -179,8 +180,8 @@ impl LpConnectionHandler { /// Handle ClientHello packet (receiver_idx=0, first packet) async fn handle_client_hello(&mut self, packet: LpPacket) -> Result<(), GatewayError> { - use nym_lp::state_machine::{LpInput, LpStateMachine}; use nym_lp::packet::LpHeader; + use nym_lp::state_machine::{LpInput, LpStateMachine}; // Extract ClientHello data let (receiver_index, client_ed25519_pubkey, salt) = match packet.message() { @@ -195,7 +196,12 @@ impl LpConnectionHandler { let client_ed25519_pubkey = nym_crypto::asymmetric::ed25519::PublicKey::from_bytes( &hello_data.client_ed25519_public_key, ) - .map_err(|e| GatewayError::LpProtocolError(format!("Invalid client Ed25519 public key: {}", e)))?; + .map_err(|e| { + GatewayError::LpProtocolError(format!( + "Invalid client Ed25519 public key: {}", + e + )) + })?; (receiver_index, client_ed25519_pubkey, hello_data.salt) } @@ -209,22 +215,26 @@ impl LpConnectionHandler { } }; - debug!("Processing ClientHello from {} (proposed receiver_index={})", self.remote_addr, receiver_index); + debug!( + "Processing ClientHello from {} (proposed receiver_index={})", + self.remote_addr, receiver_index + ); // Collision check for client-proposed receiver_index // Check both handshake_states (in-progress) and session_states (established) if self.state.handshake_states.contains_key(&receiver_index) || self.state.session_states.contains_key(&receiver_index) { - warn!("Receiver index collision: {} from {}", receiver_index, self.remote_addr); + warn!( + "Receiver index collision: {} from {}", + receiver_index, self.remote_addr + ); inc!("lp_receiver_index_collision"); // Send Collision response to tell client to retry with new receiver_index // No outer key - this is before PSK derivation - let collision_packet = LpPacket::new( - LpHeader::new(receiver_index, 0), - LpMessage::Collision, - ); + let collision_packet = + LpPacket::new(LpHeader::new(receiver_index, 0), LpMessage::Collision); self.send_lp_packet(&collision_packet, None).await?; self.emit_lifecycle_metrics(true); @@ -255,19 +265,19 @@ impl LpConnectionHandler { // Transition state machine to KKTExchange (responder waits for client's KKT request) // For responder, StartHandshake returns None (just transitions state) // For initiator, StartHandshake returns SendPacket (KKT request) - if let Some(action) = state_machine.process_input(LpInput::StartHandshake) { - if let Err(e) = action { - inc!("lp_client_hello_failed"); - return Err(GatewayError::LpHandshakeError(format!( - "StartHandshake failed: {}", - e - ))); - } + if let Some(Err(e)) = state_machine.process_input(LpInput::StartHandshake) { + inc!("lp_client_hello_failed"); + return Err(GatewayError::LpHandshakeError(format!( + "StartHandshake failed: {}", + e + ))); // Responder (gateway) gets Ok but no packet to send - we just wait for client's next packet } // Store state machine for subsequent handshake packets (KKT request with receiver_index=X) - self.state.handshake_states.insert(receiver_index, super::TimestampedState::new(state_machine)); + self.state + .handshake_states + .insert(receiver_index, super::TimestampedState::new(state_machine)); debug!( "Stored handshake state for {} (receiver_index={}) - waiting for KKT request", @@ -276,10 +286,7 @@ impl LpConnectionHandler { // Send Ack to confirm ClientHello received (packet-per-connection model) // No outer key - this is before PSK derivation - let ack_packet = LpPacket::new( - LpHeader::new(receiver_index, 0), - LpMessage::Ack, - ); + let ack_packet = LpPacket::new(LpHeader::new(receiver_index, 0), LpMessage::Ack); self.send_lp_packet(&ack_packet, None).await?; self.emit_lifecycle_metrics(true); @@ -292,7 +299,7 @@ impl LpConnectionHandler { receiver_idx: u32, packet: LpPacket, ) -> Result<(), GatewayError> { - use nym_lp::state_machine::{LpInput, LpAction}; + use nym_lp::state_machine::{LpAction, LpInput}; debug!( "Processing handshake packet from {} (receiver_idx={})", @@ -300,9 +307,16 @@ impl LpConnectionHandler { ); // Get mutable reference to state machine - let mut state_entry = self.state.handshake_states.get_mut(&receiver_idx).ok_or_else(|| { - GatewayError::LpProtocolError(format!("Handshake state not found for session {}", receiver_idx)) - })?; + let mut state_entry = self + .state + .handshake_states + .get_mut(&receiver_idx) + .ok_or_else(|| { + GatewayError::LpProtocolError(format!( + "Handshake state not found for session {}", + receiver_idx + )) + })?; let state_machine = &mut state_entry.value_mut().state; @@ -332,21 +346,42 @@ impl LpConnectionHandler { self.remote_addr, receiver_idx ); + // Get outer key for Ack encryption before releasing borrow + let outer_key = state_entry + .value() + .state + .session() + .ok() + .and_then(|s| s.outer_aead_key()); + // Move state machine to session_states (already in Transport state) // We keep the state machine (not just session) to enable // subsession/rekeying support during transport phase drop(state_entry); // Release mutable borrow - let (_receiver_idx, timestamped_state) = self.state.handshake_states.remove(&receiver_idx) - .ok_or_else(|| GatewayError::LpHandshakeError("Failed to remove handshake state".to_string()))?; + let (_receiver_idx, timestamped_state) = self + .state + .handshake_states + .remove(&receiver_idx) + .ok_or_else(|| { + GatewayError::LpHandshakeError( + "Failed to remove handshake state".to_string(), + ) + })?; - self.state.session_states.insert(receiver_idx, timestamped_state); + self.state + .session_states + .insert(receiver_idx, timestamped_state); inc!("lp_handshakes_success"); - // No response packet to send - HandshakeComplete means we're done - trace!("Moved session {} to transport mode", receiver_idx); - None + // Send Ack to confirm handshake completion to the client + let ack_packet = LpPacket::new(LpHeader::new(receiver_idx, 0), LpMessage::Ack); + trace!( + "Moved session {} to transport mode, sending Ack", + receiver_idx + ); + Some((ack_packet, outer_key)) } other => { debug!("Received action during handshake: {:?}", other); @@ -358,7 +393,11 @@ impl LpConnectionHandler { // Send response packet if needed if let Some((packet, outer_key)) = should_send { self.send_lp_packet(&packet, outer_key.as_ref()).await?; - trace!("Sent handshake response to {} (encrypted={})", self.remote_addr, outer_key.is_some()); + trace!( + "Sent handshake response to {} (encrypted={})", + self.remote_addr, + outer_key.is_some() + ); } self.emit_lifecycle_metrics(true); @@ -390,9 +429,13 @@ impl LpConnectionHandler { ); // Get state machine and process packet - let mut state_entry = self.state.session_states.get_mut(&receiver_idx).ok_or_else(|| { - GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) - })?; + let mut state_entry = self + .state + .session_states + .get_mut(&receiver_idx) + .ok_or_else(|| { + GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) + })?; // Update last activity timestamp state_entry.value().touch(); @@ -408,7 +451,10 @@ impl LpConnectionHandler { .map_err(|e| GatewayError::LpProtocolError(format!("State machine error: {}", e)))?; // Get outer key before releasing borrow - let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key()); + let outer_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key()); drop(state_entry); match action { @@ -420,13 +466,15 @@ impl LpConnectionHandler { self.remote_addr, receiver_idx ); inc!("lp_subsession_kk2_sent"); - self.send_lp_packet(&response_packet, outer_key.as_ref()).await?; + self.send_lp_packet(&response_packet, outer_key.as_ref()) + .await?; self.emit_lifecycle_metrics(true); Ok(()) } LpAction::DeliverData(data) => { // Decrypted application data - process as registration/forwarding - self.handle_decrypted_payload(receiver_idx, data.to_vec()).await + self.handle_decrypted_payload(receiver_idx, data.to_vec()) + .await } LpAction::SubsessionComplete { packet: ready_packet, @@ -437,7 +485,7 @@ impl LpConnectionHandler { self.handle_subsession_complete( receiver_idx, ready_packet, - subsession, + *subsession, new_receiver_index, outer_key, ) @@ -469,7 +517,9 @@ impl LpConnectionHandler { "LP registration request from {} (receiver_idx={}): mode={:?}", self.remote_addr, receiver_idx, request.mode ); - return self.handle_registration_request(receiver_idx, request).await; + return self + .handle_registration_request(receiver_idx, request) + .await; } // Try to deserialize as ForwardPacketData (entry gateway forwarding to exit) @@ -478,7 +528,9 @@ impl LpConnectionHandler { "LP forward request from {} (receiver_idx={}) to {}", self.remote_addr, receiver_idx, forward_data.target_lp_address ); - return self.handle_forwarding_request(receiver_idx, forward_data).await; + return self + .handle_forwarding_request(receiver_idx, forward_data) + .await; } // Neither registration nor forwarding - unknown payload type @@ -523,14 +575,20 @@ impl LpConnectionHandler { // Create new state machine from completed subsession let new_state_machine = LpStateMachine::from_subsession(subsession, new_receiver_index) .map_err(|e| { - GatewayError::LpProtocolError(format!("Failed to create session from subsession: {}", e)) + GatewayError::LpProtocolError(format!( + "Failed to create session from subsession: {}", + e + )) })?; // Check for receiver_index collision before inserting // new_receiver_index is client-generated (rand::random() in state machine). // Collisions are statistically unlikely (1 in 4 billion) but could cause DoS if exploited. if self.state.session_states.contains_key(&new_receiver_index) - || self.state.handshake_states.contains_key(&new_receiver_index) + || self + .state + .handshake_states + .contains_key(&new_receiver_index) { warn!( "Subsession receiver_index collision: {} from {}", @@ -544,9 +602,10 @@ impl LpConnectionHandler { } // Store new session under new_receiver_index - self.state - .session_states - .insert(new_receiver_index, super::TimestampedState::new(new_state_machine)); + self.state.session_states.insert( + new_receiver_index, + super::TimestampedState::new(new_state_machine), + ); // Old session is now in ReadOnlyTransport state (handled by state machine) // It will be cleaned up by TTL-based cleanup task @@ -567,9 +626,13 @@ impl LpConnectionHandler { // Acquire session lock for encryption and get outer AEAD key let (response_packet, outer_key) = { - let session_entry = self.state.session_states.get(&receiver_idx).ok_or_else(|| { - GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) - })?; + let session_entry = self + .state + .session_states + .get(&receiver_idx) + .ok_or_else(|| { + GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) + })?; // Access session via state machine for subsession support let session = session_entry .value() @@ -596,13 +659,11 @@ impl LpConnectionHandler { }; // Send response (encrypted with outer AEAD) - self.send_lp_packet(&response_packet, outer_key.as_ref()).await?; + self.send_lp_packet(&response_packet, outer_key.as_ref()) + .await?; if response.success { - info!( - "LP registration successful for {})", - self.remote_addr - ); + info!("LP registration successful for {})", self.remote_addr); } else { warn!( "LP registration failed for {}: {:?}", @@ -629,9 +690,13 @@ impl LpConnectionHandler { // Encrypt response for client and get outer AEAD key let (response_packet, outer_key) = { - let session_entry = self.state.session_states.get(&receiver_idx).ok_or_else(|| { - GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) - })?; + let session_entry = self + .state + .session_states + .get(&receiver_idx) + .ok_or_else(|| { + GatewayError::LpProtocolError(format!("Session not found: {}", receiver_idx)) + })?; // Access session via state machine for subsession support let session = session_entry .value() @@ -653,7 +718,8 @@ impl LpConnectionHandler { }; // Send encrypted response to client (encrypted with outer AEAD) - self.send_lp_packet(&response_packet, outer_key.as_ref()).await?; + self.send_lp_packet(&response_packet, outer_key.as_ref()) + .await?; debug!( "LP forwarding completed for {} (receiver_idx={})", @@ -798,8 +864,8 @@ impl LpConnectionHandler { &mut self, forward_data: ForwardPacketData, ) -> Result, GatewayError> { - use tokio::time::timeout; use std::time::Duration; + use tokio::time::timeout; inc!("lp_forward_total"); let start = std::time::Instant::now(); @@ -811,22 +877,23 @@ impl LpConnectionHandler { })?; // Connect to target gateway with timeout - let mut target_stream = match timeout(Duration::from_secs(5), TcpStream::connect(target_addr)).await { - Ok(Ok(stream)) => stream, - Ok(Err(e)) => { - inc!("lp_forward_failed"); - return Err(GatewayError::LpConnectionError(format!( - "Failed to connect to target gateway: {}", - e - ))); - } - Err(_) => { - inc!("lp_forward_failed"); - return Err(GatewayError::LpConnectionError( - "Target gateway connection timeout".to_string(), - )); - } - }; + let mut target_stream = + match timeout(Duration::from_secs(5), TcpStream::connect(target_addr)).await { + Ok(Ok(stream)) => stream, + Ok(Err(e)) => { + inc!("lp_forward_failed"); + return Err(GatewayError::LpConnectionError(format!( + "Failed to connect to target gateway: {}", + e + ))); + } + Err(_) => { + inc!("lp_forward_failed"); + return Err(GatewayError::LpConnectionError( + "Target gateway connection timeout".to_string(), + )); + } + }; debug!( "Forwarding packet to {} (target: {})", @@ -860,7 +927,10 @@ impl LpConnectionHandler { let mut len_buf = [0u8; 4]; target_stream.read_exact(&mut len_buf).await.map_err(|e| { inc!("lp_forward_failed"); - GatewayError::LpConnectionError(format!("Failed to read response length from target: {}", e)) + GatewayError::LpConnectionError(format!( + "Failed to read response length from target: {}", + e + )) })?; let response_len = u32::from_be_bytes(len_buf) as usize; @@ -881,23 +951,20 @@ impl LpConnectionHandler { .await .map_err(|e| { inc!("lp_forward_failed"); - GatewayError::LpConnectionError(format!("Failed to read response from target: {}", e)) + GatewayError::LpConnectionError(format!( + "Failed to read response from target: {}", + e + )) })?; // Record metrics let duration = start.elapsed().as_secs_f64(); - add_histogram_obs!( - "lp_forward_duration_seconds", - duration, - LP_DURATION_BUCKETS - ); + add_histogram_obs!("lp_forward_duration_seconds", duration, LP_DURATION_BUCKETS); inc!("lp_forward_success"); debug!( "Forwarding successful to {} ({} bytes response, {:.3}s)", - target_addr, - response_len, - duration + target_addr, response_len, duration ); Ok(response_buf) @@ -938,8 +1005,9 @@ impl LpConnectionHandler { self.stats.record_bytes_received(4 + packet_len); // Parse header only (for routing - header is always cleartext) - let header = parse_lp_header_only(&packet_buf) - .map_err(|e| GatewayError::LpProtocolError(format!("Failed to parse LP header: {}", e)))?; + let header = parse_lp_header_only(&packet_buf).map_err(|e| { + GatewayError::LpProtocolError(format!("Failed to parse LP header: {}", e)) + })?; Ok((packet_buf, header)) } @@ -1202,8 +1270,9 @@ mod tests { let mut handler = LpConnectionHandler::new(stream, remote_addr, state); // Two-phase: receive raw bytes + header, then parse full packet let (raw_bytes, header) = handler.receive_raw_packet().await?; - let packet = parse_lp_packet(&raw_bytes, None) - .map_err(|e| GatewayError::LpProtocolError(format!("Failed to parse packet: {}", e)))?; + let packet = parse_lp_packet(&raw_bytes, None).map_err(|e| { + GatewayError::LpProtocolError(format!("Failed to parse packet: {}", e)) + })?; Ok::<_, GatewayError>((header, packet)) }); diff --git a/gateway/src/node/lp_listener/handshake.rs b/gateway/src/node/lp_listener/handshake.rs deleted file mode 100644 index f8bec7792d1..00000000000 --- a/gateway/src/node/lp_listener/handshake.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2025 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use crate::error::GatewayError; -use nym_lp::{ - state_machine::{LpAction, LpInput, LpStateMachine}, - LpPacket, LpSession, -}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; -use tracing::*; - -/// Wrapper around the nym-lp state machine for gateway-side LP connections -pub struct LpGatewayHandshake { - state_machine: LpStateMachine, -} - -impl LpGatewayHandshake { - /// Create a new responder (gateway side) handshake - /// - /// # Arguments - /// * `receiver_index` - Client-proposed receiver_index (from ClientHello) - /// * `gateway_ed25519_keypair` - Gateway's Ed25519 identity keypair (for PSQ auth and X25519 derivation) - /// * `client_ed25519_public_key` - Client's Ed25519 public key (from ClientHello) - /// * `salt` - Salt from ClientHello (for PSK derivation) - pub fn new_responder( - receiver_index: u32, - gateway_ed25519_keypair: ( - &nym_crypto::asymmetric::ed25519::PrivateKey, - &nym_crypto::asymmetric::ed25519::PublicKey, - ), - client_ed25519_public_key: &nym_crypto::asymmetric::ed25519::PublicKey, - salt: &[u8; 32], - ) -> Result { - let state_machine = LpStateMachine::new( - receiver_index, - false, // responder - gateway_ed25519_keypair, - client_ed25519_public_key, - salt, - ) - .map_err(|e| { - GatewayError::LpHandshakeError(format!("Failed to create state machine: {}", e)) - })?; - - Ok(Self { state_machine }) - } - - /// Complete the handshake and return the established session - pub async fn complete(mut self, stream: &mut TcpStream) -> Result { - debug!("Starting LP handshake as responder"); - - // Start the handshake - if let Some(action) = self.state_machine.process_input(LpInput::StartHandshake) { - match action { - Ok(LpAction::SendPacket(packet)) => { - self.send_packet(stream, &packet).await?; - } - Ok(_) => { - // Unexpected action at this stage - return Err(GatewayError::LpHandshakeError( - "Unexpected action at handshake start".to_string(), - )); - } - Err(e) => { - return Err(GatewayError::LpHandshakeError(format!( - "Failed to start handshake: {}", - e - ))); - } - } - } - - // Continue handshake until complete - loop { - // Read incoming packet - let packet = self.receive_packet(stream).await?; - - // Process the received packet - if let Some(action) = self - .state_machine - .process_input(LpInput::ReceivePacket(packet)) - { - match action { - Ok(LpAction::SendPacket(response_packet)) => { - self.send_packet(stream, &response_packet).await?; - } - Ok(LpAction::HandshakeComplete) => { - info!("LP handshake completed successfully"); - break; - } - Ok(other) => { - debug!("Received action during handshake: {:?}", other); - } - Err(e) => { - return Err(GatewayError::LpHandshakeError(format!( - "Handshake error: {}", - e - ))); - } - } - } - } - - // Extract the session from the state machine - self.state_machine.into_session().map_err(|e| { - GatewayError::LpHandshakeError(format!("Failed to get session after handshake: {}", e)) - }) - } - - /// Send an LP packet over the stream with proper length-prefixed framing - async fn send_packet( - &self, - stream: &mut TcpStream, - packet: &LpPacket, - ) -> Result<(), GatewayError> { - use bytes::BytesMut; - use nym_lp::codec::serialize_lp_packet; - - // Serialize the packet first (None key during handshake phase) - let mut packet_buf = BytesMut::new(); - serialize_lp_packet(packet, &mut packet_buf, None).map_err(|e| { - GatewayError::LpProtocolError(format!("Failed to serialize packet: {}", e)) - })?; - - // Send 4-byte length prefix (u32 big-endian) - let len = packet_buf.len() as u32; - stream.write_all(&len.to_be_bytes()).await.map_err(|e| { - GatewayError::LpConnectionError(format!("Failed to send packet length: {}", e)) - })?; - - // Send the actual packet data - stream.write_all(&packet_buf).await.map_err(|e| { - GatewayError::LpConnectionError(format!("Failed to send packet data: {}", e)) - })?; - - stream.flush().await.map_err(|e| { - GatewayError::LpConnectionError(format!("Failed to flush stream: {}", e)) - })?; - - debug!( - "Sent LP packet ({} bytes + 4 byte header)", - packet_buf.len() - ); - Ok(()) - } - - /// Receive an LP packet from the stream with proper length-prefixed framing - async fn receive_packet(&self, stream: &mut TcpStream) -> Result { - use nym_lp::codec::parse_lp_packet; - - // Read 4-byte length prefix (u32 big-endian) - let mut len_buf = [0u8; 4]; - stream.read_exact(&mut len_buf).await.map_err(|e| { - GatewayError::LpConnectionError(format!("Failed to read packet length: {}", e)) - })?; - - let packet_len = u32::from_be_bytes(len_buf) as usize; - - // Sanity check to prevent huge allocations - const MAX_PACKET_SIZE: usize = 65536; // 64KB max - if packet_len > MAX_PACKET_SIZE { - return Err(GatewayError::LpProtocolError(format!( - "Packet size {} exceeds maximum {}", - packet_len, MAX_PACKET_SIZE - ))); - } - - // Read the actual packet data - let mut packet_buf = vec![0u8; packet_len]; - stream.read_exact(&mut packet_buf).await.map_err(|e| { - GatewayError::LpConnectionError(format!("Failed to read packet data: {}", e)) - })?; - - // Parse packet (None key during handshake phase) - let packet = parse_lp_packet(&packet_buf, None) - .map_err(|e| GatewayError::LpProtocolError(format!("Failed to parse packet: {}", e)))?; - - debug!("Received LP packet ({} bytes + 4 byte header)", packet_len); - Ok(packet) - } -} diff --git a/gateway/src/node/lp_listener/mod.rs b/gateway/src/node/lp_listener/mod.rs index ee1f3301ede..2fa64400138 100644 --- a/gateway/src/node/lp_listener/mod.rs +++ b/gateway/src/node/lp_listener/mod.rs @@ -83,7 +83,6 @@ use tokio::sync::mpsc; use tracing::*; mod handler; -mod handshake; mod messages; mod registration; @@ -269,7 +268,8 @@ impl TimestampedState { .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs(); - self.last_activity.store(now, std::sync::atomic::Ordering::Relaxed); + self.last_activity + .store(now, std::sync::atomic::Ordering::Relaxed); } /// Get age since creation @@ -283,7 +283,9 @@ impl TimestampedState { .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs(); - let last = self.last_activity.load(std::sync::atomic::Ordering::Relaxed); + let last = self + .last_activity + .load(std::sync::atomic::Ordering::Relaxed); now.saturating_sub(last) } } @@ -511,6 +513,7 @@ impl LpListener { /// /// Demoted sessions (ReadOnlyTransport) use shorter TTL since they /// only need to drain in-flight packets after subsession promotion. + #[allow(clippy::too_many_arguments)] async fn cleanup_loop( handshake_states: Arc>>, session_states: Arc>>, diff --git a/gateway/src/node/lp_listener/registration.rs b/gateway/src/node/lp_listener/registration.rs index 70fca10d70a..8d6f77ebf01 100644 --- a/gateway/src/node/lp_listener/registration.rs +++ b/gateway/src/node/lp_listener/registration.rs @@ -346,7 +346,6 @@ async fn register_wg_peer( // Create WireGuard peer with allocated IPs let mut peer = Peer::new(peer_key.clone()); - peer.preshared_key = Some(Key::new(state.local_identity.public_key().to_bytes())); peer.endpoint = None; peer.allowed_ips = vec![ format!("{client_ipv4}/32").parse()?, diff --git a/nym-gateway-probe/README.md b/nym-gateway-probe/README.md index eab109d13f8..042de230ee1 100644 --- a/nym-gateway-probe/README.md +++ b/nym-gateway-probe/README.md @@ -17,67 +17,127 @@ sudo apt install libdbus-1-dev libmnl-dev libnftnl-dev protobuf-compiler llvm-de Build required libraries and executables ```sh -# build the prober cargo build -p nym-gateway-probe ``` +## Test Modes + +The probe supports different test modes via the `--mode` flag: + +| Mode | Description | +|------|-------------| +| `mixnet` | Traditional mixnet testing - entry/exit pings + WireGuard via authenticator (default) | +| `single-hop` | LP registration + WireGuard on single gateway (no mixnet) | +| `two-hop` | Entry LP + Exit LP (nested forwarding) + WireGuard tunnel | +| `lp-only` | LP registration only - test handshake, skip WireGuard | + ## Usage +### Standard Mode (via nym-api) + +Test gateways registered in nym-api directory: + ```sh -Usage: nym-gateway-probe [OPTIONS] +# Test a specific gateway (mixnet mode) +nym-gateway-probe -g "qj3GgGYgGZZ3HkFrtD1GU9UJ5oNXME9eD2xtmPLqYYw" -Options: - -c, --config-env-file - Path pointing to an env file describing the network - -g, --entry-gateway - The specific gateway specified by ID - -n, --node - Identity of the node to test - --min-gateway-mixnet-performance - - --min-gateway-vpn-performance - - --only-wireguard - - -i, --ignore-egress-epoch-role - Disable logging during probe - --no-log - - -a, --amnezia-args - Arguments to be appended to the wireguard config enabling amnezia-wg configuration - - --netstack-download-timeout-sec - [default: 180] - --netstack-v4-dns - [default: 1.1.1.1] - --netstack-v6-dns - [default: 2606:4700:4700::1111] - --netstack-num-ping - [default: 5] - --netstack-send-timeout-sec - [default: 3] - --netstack-recv-timeout-sec - [default: 3] - --netstack-ping-hosts-v4 - [default: nymtech.net] - --netstack-ping-ips-v4 - [default: 1.1.1.1] - --netstack-ping-hosts-v6 - [default: ipv6.google.com] - --netstack-ping-ips-v6 - [default: 2001:4860:4860::8888 2606:4700:4700::1111 2620:fe::fe] - -h, --help - Print help - -V, --version - Print version +# Test with amnezia WireGuard +nym-gateway-probe -g "qj3GgGYg..." -a "jc=4\njmin=40\njmax=70\n" + +# WireGuard only (skip entry/exit ping tests) +nym-gateway-probe -g "qj3GgGYg..." --only-wireguard ``` -Examples +### Localnet Mode (run-local) + +Test gateways directly by IP/identity without nym-api: ```sh -# Run a basic probe against the node with id "qj3GgGYg..." -nym-gateway-probe -g "qj3GgGYgGZZ3HkFrtD1GU9UJ5oNXME9eD2xtmPLqYYw" +# Single-hop: LP registration + WireGuard on one gateway +nym-gateway-probe run-local \ + --entry-gateway-identity "8yGm5h2KgNwrPgRRxjT2DhXQFCnADkHVyE5FYS4LHWLC" \ + --entry-lp-address "192.168.66.6:41264" \ + --mode single-hop \ + --use-mock-ecash + +# Two-hop: Entry + Exit LP forwarding + WireGuard +nym-gateway-probe run-local \ + --entry-gateway-identity "$ENTRY_ID" \ + --entry-lp-address "192.168.66.6:41264" \ + --exit-gateway-identity "$EXIT_ID" \ + --exit-lp-address "192.168.66.7:41264" \ + --mode two-hop \ + --use-mock-ecash + +# LP-only: Test handshake and registration only +nym-gateway-probe run-local \ + --entry-gateway-identity "$GATEWAY_ID" \ + --entry-lp-address "localhost:41264" \ + --mode lp-only \ + --use-mock-ecash +``` + +**Note:** `--use-mock-ecash` requires gateways started with `--lp-use-mock-ecash`. + +### Split Network Configuration + +For docker/container setups where entry and exit are on different networks: + +```sh +# Entry reachable from host, exit only reachable from entry's internal network +nym-gateway-probe run-local \ + --entry-gateway-identity "$ENTRY_ID" \ + --entry-lp-address "192.168.66.6:41264" \ # Host → Entry + --exit-gateway-identity "$EXIT_ID" \ + --exit-lp-address "172.18.0.5:41264" \ # Entry → Exit (internal) + --mode two-hop \ + --use-mock-ecash +``` + +## CLI Reference + +``` +Usage: nym-gateway-probe [OPTIONS] [COMMAND] + +Commands: + run-local Run probe in localnet mode (direct IP, no nym-api) + +Options: + -c, --config-env-file Path to env file describing the network + -g, --entry-gateway Entry gateway identity (base58) + -n, --node Node to test (defaults to entry gateway) + --gateway-ip Query gateway directly by IP (skip nym-api) + --exit-gateway-ip Exit gateway IP for two-hop testing + --mode Test mode: mixnet, single-hop, two-hop, lp-only + --only-wireguard Skip ping tests, only test WireGuard + --only-lp-registration Test LP registration only (legacy flag) + --test-lp-wg Test LP + WireGuard (legacy flag) + -a, --amnezia-args Amnezia WireGuard config arguments + --no-log Disable logging + -h, --help Print help + -V, --version Print version + +Localnet Options (run-local): + --entry-gateway-identity Entry gateway Ed25519 identity + --entry-lp-address Entry gateway LP listener address + --exit-gateway-identity Exit gateway Ed25519 identity + --exit-lp-address Exit gateway LP listener address + --use-mock-ecash Use mock credentials (dev only) +``` + +## Output + +The probe outputs JSON with test results: -# Run a probe against the node with id "qj3GgGYg..." using amnezia with junk packets enabled. -nym-gateway-probe -g "qj3GgGYgGZZ3HkFrtD1GU9UJ5oNXME9eD2xtmPLqYYw" -a "jc=4\njmin=40\njmax=70\n" +```json +{ + "node": "gateway-identity", + "used_entry": "entry-gateway-identity", + "outcome": { + "as_entry": { "can_connect": true, "can_route": true }, + "as_exit": { "can_connect": true, "can_route_ip_v4": true, "can_route_ip_v6": true }, + "wg": { "can_register": true, "can_handshake_v4": true, "can_handshake_v6": true }, + "lp": { "can_connect": true, "can_handshake": true, "can_register": true } + } +} ``` diff --git a/nym-gateway-probe/build.rs b/nym-gateway-probe/build.rs index 9588105290a..43af6756508 100644 --- a/nym-gateway-probe/build.rs +++ b/nym-gateway-probe/build.rs @@ -68,7 +68,9 @@ fn build_go() -> anyhow::Result<()> { .arg(binary_out_path) .arg("-buildmode") .arg("c-archive") + // Include all Go source files in the package (except tests) .arg("lib.go") + .arg("udp_forwarder.go") .spawn()?; let status = child.wait()?; if !status.success() { diff --git a/nym-gateway-probe/netstack_ping/lib.go b/nym-gateway-probe/netstack_ping/lib.go index 83229fa9a6d..7d04b7cbf22 100644 --- a/nym-gateway-probe/netstack_ping/lib.go +++ b/nym-gateway-probe/netstack_ping/lib.go @@ -135,6 +135,277 @@ func wgFreePtr(ptr unsafe.Pointer) { C.free(ptr) } +// TwoHopNetstackRequest contains configuration for two-hop WireGuard tunneling. +// Traffic flows: Client -> Entry WG Tunnel -> UDP Forwarder -> Exit WG Tunnel -> Internet +type TwoHopNetstackRequest struct { + // Entry tunnel configuration (connects to entry gateway) + EntryWgIp string `json:"entry_wg_ip"` + EntryPrivateKey string `json:"entry_private_key"` + EntryPublicKey string `json:"entry_public_key"` + EntryEndpoint string `json:"entry_endpoint"` + EntryAwgArgs string `json:"entry_awg_args"` + + // Exit tunnel configuration (connects via forwarder through entry) + ExitWgIp string `json:"exit_wg_ip"` + ExitPrivateKey string `json:"exit_private_key"` + ExitPublicKey string `json:"exit_public_key"` + ExitEndpoint string `json:"exit_endpoint"` // Actual exit gateway endpoint (forwarded via entry) + ExitAwgArgs string `json:"exit_awg_args"` + + // Test parameters (same as single-hop) + Dns string `json:"dns"` + IpVersion uint8 `json:"ip_version"` + PingHosts []string `json:"ping_hosts"` + PingIps []string `json:"ping_ips"` + NumPing uint8 `json:"num_ping"` + SendTimeoutSec uint64 `json:"send_timeout_sec"` + RecvTimeoutSec uint64 `json:"recv_timeout_sec"` + DownloadTimeoutSec uint64 `json:"download_timeout_sec"` +} + +// Default port that exit WG tunnel uses to send traffic to the forwarder. +// The forwarder only accepts packets from this port on loopback. +const DEFAULT_EXIT_WG_CLIENT_PORT uint16 = 54001 + +// Entry tunnel MTU (outer tunnel) +const ENTRY_MTU = 1420 + +// Exit tunnel MTU (must be smaller due to double encapsulation) +const EXIT_MTU = 1340 + +//export wgPingTwoHop +func wgPingTwoHop(cReq *C.char) *C.char { + reqStr := C.GoString(cReq) + + var req TwoHopNetstackRequest + err := json.Unmarshal([]byte(reqStr), &req) + if err != nil { + log.Printf("Failed to parse two-hop request: %s", err) + return jsonError(err) + } + + response, err := pingTwoHop(req) + if err != nil { + log.Printf("Failed to ping (two-hop): %s", err) + return jsonError(err) + } + + return jsonResponse(response) +} + +func pingTwoHop(req TwoHopNetstackRequest) (NetstackResponse, error) { + log.Printf("=== Two-Hop WireGuard Probe ===") + log.Printf("Entry endpoint: %s", req.EntryEndpoint) + log.Printf("Entry WG IP: %s", req.EntryWgIp) + log.Printf("Exit endpoint: %s (via entry forwarding)", req.ExitEndpoint) + log.Printf("Exit WG IP: %s", req.ExitWgIp) + log.Printf("IP version: %d", req.IpVersion) + + response := NetstackResponse{false, false, 0, 0, 0, 0, false, "", 0, 0, 0, ""} + + // Parse the exit endpoint to determine IP version for forwarder + exitEndpoint, err := netip.ParseAddrPort(req.ExitEndpoint) + if err != nil { + return response, fmt.Errorf("failed to parse exit endpoint: %w", err) + } + + // ============================================ + // STEP 1: Create entry tunnel (netstack) + // ============================================ + log.Printf("Creating entry tunnel (MTU=%d)...", ENTRY_MTU) + + entryTun, entryTnet, err := netstack.CreateNetTUN( + []netip.Addr{netip.MustParseAddr(req.EntryWgIp)}, + []netip.Addr{netip.MustParseAddr(req.Dns)}, + ENTRY_MTU) + if err != nil { + return response, fmt.Errorf("failed to create entry tunnel: %w", err) + } + + entryLogger := device.NewLogger(device.LogLevelError, "entry: ") + entryDev := device.NewDevice(entryTun, conn.NewDefaultBind(), entryLogger) + defer entryDev.Close() + + // Configure entry device + var entryIpc strings.Builder + entryIpc.WriteString("private_key=") + entryIpc.WriteString(req.EntryPrivateKey) + if req.EntryAwgArgs != "" { + awg := strings.ReplaceAll(req.EntryAwgArgs, "\\n", "\n") + entryIpc.WriteString(fmt.Sprintf("\n%s", awg)) + } + entryIpc.WriteString("\npublic_key=") + entryIpc.WriteString(req.EntryPublicKey) + entryIpc.WriteString("\nendpoint=") + entryIpc.WriteString(req.EntryEndpoint) + // Entry tunnel routes all traffic (the exit endpoint IP goes through it) + entryIpc.WriteString("\nallowed_ip=0.0.0.0/0") + entryIpc.WriteString("\nallowed_ip=::/0\n") + + if err := entryDev.IpcSet(entryIpc.String()); err != nil { + return response, fmt.Errorf("failed to configure entry device: %w", err) + } + + if err := entryDev.Up(); err != nil { + return response, fmt.Errorf("failed to bring up entry device: %w", err) + } + log.Printf("Entry tunnel up") + + // ============================================ + // STEP 2: Create UDP forwarder + // ============================================ + log.Printf("Creating UDP forwarder (exit endpoint: %s)...", exitEndpoint.String()) + + forwarderConfig := UDPForwarderConfig{ + ListenPort: 0, // Dynamic port assignment + ClientPort: DEFAULT_EXIT_WG_CLIENT_PORT, + Endpoint: exitEndpoint, + } + + forwarder, err := NewUDPForwarder(forwarderConfig, entryTnet, entryLogger) + if err != nil { + return response, fmt.Errorf("failed to create UDP forwarder: %w", err) + } + defer forwarder.Close() + + forwarderAddr := forwarder.GetListenAddr() + log.Printf("UDP forwarder listening on: %s", forwarderAddr.String()) + + // ============================================ + // STEP 3: Create exit tunnel (netstack) + // ============================================ + log.Printf("Creating exit tunnel (MTU=%d)...", EXIT_MTU) + + exitTun, exitTnet, err := netstack.CreateNetTUN( + []netip.Addr{netip.MustParseAddr(req.ExitWgIp)}, + []netip.Addr{netip.MustParseAddr(req.Dns)}, + EXIT_MTU) + if err != nil { + return response, fmt.Errorf("failed to create exit tunnel: %w", err) + } + + exitLogger := device.NewLogger(device.LogLevelError, "exit: ") + exitDev := device.NewDevice(exitTun, conn.NewDefaultBind(), exitLogger) + defer exitDev.Close() + + // Configure exit device - endpoint is the forwarder, NOT the actual exit gateway + var exitIpc strings.Builder + exitIpc.WriteString("private_key=") + exitIpc.WriteString(req.ExitPrivateKey) + // Set listen_port so the forwarder knows which port to accept packets from + exitIpc.WriteString(fmt.Sprintf("\nlisten_port=%d", DEFAULT_EXIT_WG_CLIENT_PORT)) + if req.ExitAwgArgs != "" { + awg := strings.ReplaceAll(req.ExitAwgArgs, "\\n", "\n") + exitIpc.WriteString(fmt.Sprintf("\n%s", awg)) + } + exitIpc.WriteString("\npublic_key=") + exitIpc.WriteString(req.ExitPublicKey) + // IMPORTANT: endpoint is the local forwarder, not the actual exit gateway! + exitIpc.WriteString("\nendpoint=") + exitIpc.WriteString(forwarderAddr.String()) + if req.IpVersion == 4 { + exitIpc.WriteString("\nallowed_ip=0.0.0.0/0\n") + } else { + exitIpc.WriteString("\nallowed_ip=::/0\n") + } + + if err := exitDev.IpcSet(exitIpc.String()); err != nil { + return response, fmt.Errorf("failed to configure exit device: %w", err) + } + + if err := exitDev.Up(); err != nil { + return response, fmt.Errorf("failed to bring up exit device: %w", err) + } + log.Printf("Exit tunnel up (via forwarder)") + + // If we got here, both tunnels and forwarder are set up + response.CanHandshake = true + log.Printf("Two-hop tunnel setup complete!") + + // ============================================ + // STEP 4: Run tests through exit tunnel + // ============================================ + log.Printf("Running tests through exit tunnel...") + + // Ping hosts (DNS resolution test) + for _, host := range req.PingHosts { + consecutiveFailures := 0 + maxConsecutiveFailures := 3 + + for i := uint8(0); i < req.NumPing; i++ { + log.Printf("Pinging %s seq=%d (via two-hop)", host, i) + response.SentHosts += 1 + rt, err := sendPing(host, i, req.SendTimeoutSec, req.RecvTimeoutSec, exitTnet, req.IpVersion) + if err != nil { + log.Printf("Failed to send ping: %v", err) + consecutiveFailures++ + if consecutiveFailures >= maxConsecutiveFailures { + log.Printf("Too many consecutive failures (%d), stopping ping attempts for %s", consecutiveFailures, host) + break + } + continue + } + consecutiveFailures = 0 + response.ReceivedHosts += 1 + response.CanResolveDns = true + log.Printf("Ping latency: %v", rt) + } + } + + // Ping IPs (direct connectivity test) + for _, ip := range req.PingIps { + consecutiveFailures := 0 + maxConsecutiveFailures := 3 + + for i := uint8(0); i < req.NumPing; i++ { + log.Printf("Pinging %s seq=%d (via two-hop)", ip, i) + response.SentIps += 1 + rt, err := sendPing(ip, i, req.SendTimeoutSec, req.RecvTimeoutSec, exitTnet, req.IpVersion) + if err != nil { + log.Printf("Failed to send ping: %v", err) + consecutiveFailures++ + if consecutiveFailures >= maxConsecutiveFailures { + log.Printf("Too many consecutive failures (%d), stopping ping attempts for %s", consecutiveFailures, ip) + break + } + } else { + consecutiveFailures = 0 + response.ReceivedIps += 1 + log.Printf("Ping latency: %v", rt) + } + + if i < req.NumPing-1 { + time.Sleep(5 * time.Second) + } + } + } + + // Download test + var urlsToTry []string + if req.IpVersion == 4 { + urlsToTry = fileUrls + } else { + urlsToTry = fileUrlsV6 + } + + fileContent, downloadDuration, usedURL, err := downloadFileWithRetry(urlsToTry, req.DownloadTimeoutSec, exitTnet) + if err != nil { + log.Printf("Failed to download file from any URL: %v", err) + response.DownloadError = err.Error() + } else { + log.Printf("Downloaded file content length: %.2f MB", float64(len(fileContent))/1024/1024) + log.Printf("Download duration: %v", downloadDuration) + response.DownloadedFileSizeBytes = uint64(len(fileContent)) + } + + response.DownloadDurationSec = uint64(downloadDuration.Seconds()) + response.DownloadDurationMilliseconds = uint64(downloadDuration.Milliseconds()) + response.DownloadedFile = usedURL + + log.Printf("=== Two-Hop Probe Complete ===") + return response, nil +} + func ping(req NetstackRequestGo) (NetstackResponse, error) { fmt.Printf("Endpoint: %s\n", req.Endpoint) fmt.Printf("WireGuard IP: %s\n", req.WgIp) diff --git a/nym-gateway-probe/netstack_ping/udp_forwarder.go b/nym-gateway-probe/netstack_ping/udp_forwarder.go new file mode 100644 index 00000000000..af4fdf03386 --- /dev/null +++ b/nym-gateway-probe/netstack_ping/udp_forwarder.go @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-3.0-only + * + * Copyright 2024 - Nym Technologies SA + * + * UDP forwarder for two-hop WireGuard tunneling. + * Copied from nym-vpn-client/wireguard/libwg/forwarders/udp.go + */ + +package main + +import ( + "log" + "net" + "net/netip" + "sync" + "time" + + "github.com/amnezia-vpn/amneziawg-go/device" + "github.com/amnezia-vpn/amneziawg-go/tun/netstack" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" +) + +const UDP_WRITE_TIMEOUT = time.Duration(5) * time.Second +const MAX_UDP_DATAGRAM_LEN = 65535 + +type UDPForwarderConfig struct { + // Listen port for incoming UDP traffic. + // For IPv4 endpoint, the listening port is bound to 127.0.0.1, for IPv6 it's ::1. + ListenPort uint16 + + // Client port on loopback from which the incoming connection will be received. + // Only packets from this port will be passed through to the endpoint. + ClientPort uint16 + + // Endpoint to connect to over netstack + Endpoint netip.AddrPort +} + +// UDP forwarder that creates a bidirectional in-tunnel connection between a local and remote UDP endpoints +type UDPForwarder struct { + logger *device.Logger + + // Netstack tunnel + tnet *netstack.Net + + // UDP listener that receives inbound traffic piped to the remote endpoint + listener *net.UDPConn + + // Outbound connection to the remote endpoint over the entry tunnel + outbound *gonet.UDPConn + + // Wait group used to signal when all goroutines have finished execution + waitGroup *sync.WaitGroup + + // In netstack mode, conn.NewDefaultBind() doesn't honor listen_port IPC setting, + // so we learn the actual client address from the first inbound packet. + // This is protected by clientAddrMu and signaled via clientAddrCond. + clientAddrMu sync.Mutex + clientAddrCond *sync.Cond + learnedClient *net.UDPAddr +} + +func NewUDPForwarder(config UDPForwarderConfig, tnet *netstack.Net, logger *device.Logger) (*UDPForwarder, error) { + var listenAddr *net.UDPAddr + var clientAddr *net.UDPAddr + + // Use the same ip protocol family as endpoint + if config.Endpoint.Addr().Is4() { + loopback := netip.AddrFrom4([4]byte{127, 0, 0, 1}) + listenAddr = net.UDPAddrFromAddrPort(netip.AddrPortFrom(loopback, config.ListenPort)) + clientAddr = net.UDPAddrFromAddrPort(netip.AddrPortFrom(loopback, config.ClientPort)) + } else { + listenAddr = net.UDPAddrFromAddrPort(netip.AddrPortFrom(netip.IPv6Loopback(), config.ListenPort)) + clientAddr = net.UDPAddrFromAddrPort(netip.AddrPortFrom(netip.IPv6Loopback(), config.ClientPort)) + } + + listener, err := net.ListenUDP("udp", listenAddr) + if err != nil { + return nil, err + } + + outbound, err := tnet.DialUDPAddrPort(netip.AddrPort{}, config.Endpoint) + if err != nil { + listener.Close() + return nil, err + } + + waitGroup := &sync.WaitGroup{} + wrapper := &UDPForwarder{ + logger: logger, + tnet: tnet, + listener: listener, + outbound: outbound, + waitGroup: waitGroup, + learnedClient: nil, + } + wrapper.clientAddrCond = sync.NewCond(&wrapper.clientAddrMu) + + waitGroup.Add(2) + go wrapper.routineHandleInbound(listener, outbound, clientAddr) + go wrapper.routineHandleOutbound(listener, outbound, clientAddr) + + return wrapper, nil +} + +func (w *UDPForwarder) GetListenAddr() net.Addr { + return w.listener.LocalAddr() +} + +func (w *UDPForwarder) Close() { + // Close all connections. This should release any blocking ReadFromUDP() calls + w.listener.Close() + w.outbound.Close() + + // Wait for all routines to complete + w.waitGroup.Wait() +} + +func (w *UDPForwarder) Wait() { + w.waitGroup.Wait() +} + +func (w *UDPForwarder) routineHandleInbound(inbound *net.UDPConn, outbound *gonet.UDPConn, clientAddr *net.UDPAddr) { + defer w.waitGroup.Done() + defer outbound.Close() + + inboundBuffer := make([]byte, MAX_UDP_DATAGRAM_LEN) + + w.logger.Verbosef("udpforwarder(inbound): listening on %s (proxy to %s)", inbound.LocalAddr().String(), outbound.RemoteAddr().String()) + defer w.logger.Verbosef("udpforwarder(inbound): closed") + + for { + // Receive the WireGuard packet from local port + bytesRead, senderAddr, err := inbound.ReadFromUDP(inboundBuffer) + if err != nil { + w.logger.Errorf("udpforwarder(inbound): %s", err.Error()) + return + } + + log.Printf("udpforwarder(inbound): received %d bytes from %s", bytesRead, senderAddr.String()) + + // Only accept packets from loopback + if !senderAddr.IP.IsLoopback() { + log.Printf("udpforwarder(inbound): drop packet from non-loopback: %s", senderAddr.String()) + continue + } + + // Learn the client address from the first packet and notify the outbound handler + // In netstack mode, conn.NewDefaultBind() doesn't honor listen_port IPC setting, + // so we learn the actual client address from the first inbound packet. + w.clientAddrMu.Lock() + if w.learnedClient == nil { + w.learnedClient = senderAddr + log.Printf("udpforwarder(inbound): learned client addr: %s", w.learnedClient.String()) + w.clientAddrCond.Broadcast() // Signal outbound handler + } + learnedPort := w.learnedClient.Port + w.clientAddrMu.Unlock() + + // Drop packet from unknown sender (different port than the learned client) + if senderAddr.Port != learnedPort { + log.Printf("udpforwarder(inbound): drop packet from unknown sender: %s, expected port: %d.", senderAddr.String(), learnedPort) + continue + } + + log.Printf("udpforwarder(inbound): forwarding %d bytes to exit gateway", bytesRead) + + // Set write timeout for outbound + deadline := time.Now().Add(UDP_WRITE_TIMEOUT) + err = outbound.SetWriteDeadline(deadline) + if err != nil { + w.logger.Errorf("udpforwarder(inbound): %s", err.Error()) + return + } + + // Forward the packet over the outbound connection via another WireGuard tunnel + bytesWritten, err := outbound.Write(inboundBuffer[:bytesRead]) + if err != nil { + w.logger.Errorf("udpforwarder(inbound): %s", err.Error()) + return + } + + if bytesWritten != bytesRead { + w.logger.Errorf("udpforwarder(inbound): wrote %d bytes, expected %d", bytesWritten, bytesRead) + } + } +} + +func (w *UDPForwarder) routineHandleOutbound(inbound *net.UDPConn, outbound *gonet.UDPConn, clientAddr *net.UDPAddr) { + defer w.waitGroup.Done() + defer inbound.Close() + + remoteAddr := outbound.RemoteAddr().(*net.UDPAddr) + w.logger.Verbosef("udpforwarder(outbound): dial %s", remoteAddr.String()) + defer w.logger.Verbosef("udpforwarder(outbound): closed") + + outboundBuffer := make([]byte, MAX_UDP_DATAGRAM_LEN) + + for { + // Receive WireGuard packet from remote server + bytesRead, senderAddr, err := outbound.ReadFrom(outboundBuffer) + if err != nil { + w.logger.Errorf("udpforwarder(outbound): %s", err.Error()) + return + } + // Cast net.Addr to net.UDPAddr + senderUDPAddr := senderAddr.(*net.UDPAddr) + + log.Printf("udpforwarder(outbound): received %d bytes from %s", bytesRead, senderUDPAddr.String()) + + // Drop packet from unknown sender. + if !senderUDPAddr.IP.Equal(remoteAddr.IP) || senderUDPAddr.Port != remoteAddr.Port { + log.Printf("udpforwarder(outbound): drop packet from unknown sender: %s, expected: %s", senderUDPAddr.String(), remoteAddr.String()) + continue + } + + // Wait for the learned client address from the inbound handler + // This ensures we send responses to the actual client port (which may differ from expected) + w.clientAddrMu.Lock() + for w.learnedClient == nil { + w.clientAddrCond.Wait() + } + targetClient := w.learnedClient + w.clientAddrMu.Unlock() + + log.Printf("udpforwarder(outbound): forwarding %d bytes to client %s", bytesRead, targetClient.String()) + + // Set write timeout for inbound + deadline := time.Now().Add(UDP_WRITE_TIMEOUT) + err = inbound.SetWriteDeadline(deadline) + if err != nil { + w.logger.Errorf("udpforwarder(outbound): %s", err.Error()) + return + } + + // Forward packet from remote to local client (using learned address) + bytesWritten, err := inbound.WriteToUDP(outboundBuffer[:bytesRead], targetClient) + if err != nil { + w.logger.Errorf("udpforwarder(outbound): %s", err.Error()) + return + } + + if bytesWritten != bytesRead { + w.logger.Errorf("udpforwarder(outbound): wrote %d bytes, expected %d", bytesWritten, bytesRead) + } + } +} diff --git a/nym-gateway-probe/src/common/mod.rs b/nym-gateway-probe/src/common/mod.rs new file mode 100644 index 00000000000..930623615d7 --- /dev/null +++ b/nym-gateway-probe/src/common/mod.rs @@ -0,0 +1,13 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +//! Common utilities shared across test modes. +//! +//! This module contains shared functionality used by multiple test modes: +//! - WireGuard tunnel testing via netstack + +pub mod wireguard; + +pub use wireguard::{ + TwoHopWgTunnelConfig, WgTunnelConfig, run_tunnel_tests, run_two_hop_tunnel_tests, +}; diff --git a/nym-gateway-probe/src/common/wireguard.rs b/nym-gateway-probe/src/common/wireguard.rs new file mode 100644 index 00000000000..3d58d68a539 --- /dev/null +++ b/nym-gateway-probe/src/common/wireguard.rs @@ -0,0 +1,307 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +//! Shared WireGuard tunnel testing via netstack. +//! +//! This module provides common functionality for testing WireGuard tunnels +//! that is shared between different test modes (authenticator-based and LP-based). + +use nym_config::defaults::{WG_METADATA_PORT, WG_TUN_DEVICE_IP_ADDRESS_V4}; +use tracing::{error, info}; + +use crate::NetstackArgs; +use crate::netstack::{ + NetstackRequest, NetstackRequestGo, NetstackResult, TwoHopNetstackRequestGo, +}; +use crate::types::WgProbeResults; + +/// Safe division that returns 0.0 when divisor is 0 (instead of NaN/Inf) +fn safe_ratio(received: u16, sent: u16) -> f32 { + if sent == 0 { + 0.0 + } else { + received as f32 / sent as f32 + } +} + +/// WireGuard tunnel configuration for netstack testing. +/// +/// Contains all the parameters needed to establish and test a WireGuard tunnel. +pub struct WgTunnelConfig { + /// Client's private IPv4 address in the tunnel + pub private_ipv4: String, + /// Client's private IPv6 address in the tunnel + pub private_ipv6: String, + /// Client's WireGuard private key (hex encoded) + pub private_key_hex: String, + /// Gateway's WireGuard public key (hex encoded) + pub public_key_hex: String, + /// WireGuard endpoint address (gateway_ip:port) + pub endpoint: String, +} + +impl WgTunnelConfig { + /// Create a new tunnel configuration. + pub fn new( + private_ipv4: impl Into, + private_ipv6: impl Into, + private_key_hex: impl Into, + public_key_hex: impl Into, + endpoint: impl Into, + ) -> Self { + Self { + private_ipv4: private_ipv4.into(), + private_ipv6: private_ipv6.into(), + private_key_hex: private_key_hex.into(), + public_key_hex: public_key_hex.into(), + endpoint: endpoint.into(), + } + } +} + +/// Run WireGuard tunnel connectivity tests using netstack. +/// +/// This function tests both IPv4 and IPv6 connectivity through the WireGuard tunnel: +/// - DNS resolution +/// - ICMP ping to specified hosts and IPs +/// - Optional download test +/// +/// Results are written directly into the provided `wg_outcome` to avoid field-by-field +/// copying at call sites. +/// +/// # Arguments +/// * `config` - WireGuard tunnel configuration +/// * `netstack_args` - Netstack test parameters (DNS, hosts to ping, timeouts, etc.) +/// * `awg_args` - Amnezia WireGuard arguments (empty string for standard WG) +/// * `wg_outcome` - Mutable reference to write test results into +// This function extracts the shared netstack testing logic from +// wg_probe() and wg_probe_lp() to eliminate code duplication. +pub fn run_tunnel_tests( + config: &WgTunnelConfig, + netstack_args: &NetstackArgs, + awg_args: &str, + wg_outcome: &mut WgProbeResults, +) { + // Build the netstack request + let netstack_request = NetstackRequest::new( + &config.private_ipv4, + &config.private_ipv6, + &config.private_key_hex, + &config.public_key_hex, + &config.endpoint, + &format!("http://{WG_TUN_DEVICE_IP_ADDRESS_V4}:{WG_METADATA_PORT}"), + netstack_args.netstack_download_timeout_sec, + awg_args, + netstack_args.clone(), + ); + + // Perform IPv4 ping test + info!("Testing IPv4 tunnel connectivity..."); + let ipv4_request = NetstackRequestGo::from_rust_v4(&netstack_request); + + match crate::netstack::ping(&ipv4_request) { + Ok(NetstackResult::Response(netstack_response_v4)) => { + info!( + "WireGuard probe response for IPv4: {:#?}", + netstack_response_v4 + ); + wg_outcome.can_query_metadata_v4 = netstack_response_v4.can_query_metadata; + wg_outcome.can_handshake_v4 = netstack_response_v4.can_handshake; + wg_outcome.can_resolve_dns_v4 = netstack_response_v4.can_resolve_dns; + wg_outcome.ping_hosts_performance_v4 = safe_ratio( + netstack_response_v4.received_hosts, + netstack_response_v4.sent_hosts, + ); + wg_outcome.ping_ips_performance_v4 = safe_ratio( + netstack_response_v4.received_ips, + netstack_response_v4.sent_ips, + ); + + wg_outcome.download_duration_sec_v4 = netstack_response_v4.download_duration_sec; + wg_outcome.download_duration_milliseconds_v4 = + netstack_response_v4.download_duration_milliseconds; + wg_outcome.downloaded_file_size_bytes_v4 = + netstack_response_v4.downloaded_file_size_bytes; + wg_outcome.downloaded_file_v4 = netstack_response_v4.downloaded_file; + wg_outcome.download_error_v4 = netstack_response_v4.download_error; + } + Ok(NetstackResult::Error { error }) => { + error!("Netstack runtime error (IPv4): {error}") + } + Err(error) => { + error!("Internal error (IPv4): {error}") + } + } + + // Perform IPv6 ping test + info!("Testing IPv6 tunnel connectivity..."); + let ipv6_request = NetstackRequestGo::from_rust_v6(&netstack_request); + + match crate::netstack::ping(&ipv6_request) { + Ok(NetstackResult::Response(netstack_response_v6)) => { + info!( + "WireGuard probe response for IPv6: {:#?}", + netstack_response_v6 + ); + wg_outcome.can_handshake_v6 = netstack_response_v6.can_handshake; + wg_outcome.can_resolve_dns_v6 = netstack_response_v6.can_resolve_dns; + wg_outcome.ping_hosts_performance_v6 = safe_ratio( + netstack_response_v6.received_hosts, + netstack_response_v6.sent_hosts, + ); + wg_outcome.ping_ips_performance_v6 = safe_ratio( + netstack_response_v6.received_ips, + netstack_response_v6.sent_ips, + ); + + wg_outcome.download_duration_sec_v6 = netstack_response_v6.download_duration_sec; + wg_outcome.download_duration_milliseconds_v6 = + netstack_response_v6.download_duration_milliseconds; + wg_outcome.downloaded_file_size_bytes_v6 = + netstack_response_v6.downloaded_file_size_bytes; + wg_outcome.downloaded_file_v6 = netstack_response_v6.downloaded_file; + wg_outcome.download_error_v6 = netstack_response_v6.download_error; + } + Ok(NetstackResult::Error { error }) => { + error!("Netstack runtime error (IPv6): {error}") + } + Err(error) => { + error!("Internal error (IPv6): {error}") + } + } +} + +/// Two-hop WireGuard tunnel configuration for nested tunnel testing. +/// +/// Traffic flows: Exit tunnel -> UDP Forwarder -> Entry tunnel -> Exit Gateway -> Internet +// This is used for LP two-hop mode where traffic must go through entry gateway +// to reach exit gateway. The forwarder bridges the two netstack tunnels on localhost. +pub struct TwoHopWgTunnelConfig { + // Entry tunnel (outer, connects directly to entry gateway) + /// Entry client's private IPv4 address in the tunnel + pub entry_private_ipv4: String, + /// Entry client's WireGuard private key (hex encoded) + pub entry_private_key_hex: String, + /// Entry gateway's WireGuard public key (hex encoded) + pub entry_public_key_hex: String, + /// Entry WireGuard endpoint address (entry_gateway_ip:port) + pub entry_endpoint: String, + /// Entry Amnezia WG args (empty for standard WG) + pub entry_awg_args: String, + + // Exit tunnel (inner, connects via forwarder through entry) + /// Exit client's private IPv4 address in the tunnel + pub exit_private_ipv4: String, + /// Exit client's WireGuard private key (hex encoded) + pub exit_private_key_hex: String, + /// Exit gateway's WireGuard public key (hex encoded) + pub exit_public_key_hex: String, + /// Exit WireGuard endpoint address (exit_gateway_ip:port, forwarded via entry) + pub exit_endpoint: String, + /// Exit Amnezia WG args (empty for standard WG) + pub exit_awg_args: String, +} + +impl TwoHopWgTunnelConfig { + /// Create a new two-hop tunnel configuration. + #[allow(clippy::too_many_arguments)] + pub fn new( + entry_private_ipv4: impl Into, + entry_private_key_hex: impl Into, + entry_public_key_hex: impl Into, + entry_endpoint: impl Into, + entry_awg_args: impl Into, + exit_private_ipv4: impl Into, + exit_private_key_hex: impl Into, + exit_public_key_hex: impl Into, + exit_endpoint: impl Into, + exit_awg_args: impl Into, + ) -> Self { + Self { + entry_private_ipv4: entry_private_ipv4.into(), + entry_private_key_hex: entry_private_key_hex.into(), + entry_public_key_hex: entry_public_key_hex.into(), + entry_endpoint: entry_endpoint.into(), + entry_awg_args: entry_awg_args.into(), + exit_private_ipv4: exit_private_ipv4.into(), + exit_private_key_hex: exit_private_key_hex.into(), + exit_public_key_hex: exit_public_key_hex.into(), + exit_endpoint: exit_endpoint.into(), + exit_awg_args: exit_awg_args.into(), + } + } +} + +/// Run two-hop WireGuard tunnel connectivity tests using netstack. +/// +/// This function tests connectivity through nested WireGuard tunnels: +/// - Entry tunnel connects directly to entry gateway +/// - UDP forwarder bridges entry and exit tunnels on localhost +/// - Exit tunnel sends traffic via forwarder -> entry tunnel -> exit gateway +/// - Tests (DNS, ping, download) run through the exit tunnel +/// +/// # Arguments +/// * `config` - Two-hop WireGuard tunnel configuration +/// * `netstack_args` - Netstack test parameters (DNS, hosts to ping, timeouts, etc.) +/// * `wg_outcome` - Mutable reference to write test results into +// Currently only tests IPv4. IPv6 support can be added later if needed. +pub fn run_two_hop_tunnel_tests( + config: &TwoHopWgTunnelConfig, + netstack_args: &NetstackArgs, + wg_outcome: &mut WgProbeResults, +) { + // Build the two-hop netstack request for IPv4 + let request = TwoHopNetstackRequestGo { + // Entry tunnel config + entry_wg_ip: config.entry_private_ipv4.clone(), + entry_private_key: config.entry_private_key_hex.clone(), + entry_public_key: config.entry_public_key_hex.clone(), + entry_endpoint: config.entry_endpoint.clone(), + entry_awg_args: config.entry_awg_args.clone(), + + // Exit tunnel config + exit_wg_ip: config.exit_private_ipv4.clone(), + exit_private_key: config.exit_private_key_hex.clone(), + exit_public_key: config.exit_public_key_hex.clone(), + exit_endpoint: config.exit_endpoint.clone(), + exit_awg_args: config.exit_awg_args.clone(), + + // Test parameters (use IPv4 config) + dns: netstack_args.netstack_v4_dns.clone(), + ip_version: 4, + ping_hosts: netstack_args.netstack_ping_hosts_v4.clone(), + ping_ips: netstack_args.netstack_ping_ips_v4.clone(), + num_ping: netstack_args.netstack_num_ping, + send_timeout_sec: netstack_args.netstack_send_timeout_sec, + recv_timeout_sec: netstack_args.netstack_recv_timeout_sec, + download_timeout_sec: netstack_args.netstack_download_timeout_sec, + }; + + info!("Testing two-hop IPv4 tunnel connectivity..."); + info!(" Entry endpoint: {}", config.entry_endpoint); + info!(" Exit endpoint (via forwarder): {}", config.exit_endpoint); + + match crate::netstack::ping_two_hop(&request) { + Ok(NetstackResult::Response(response)) => { + info!("Two-hop WireGuard probe response (IPv4): {:#?}", response); + wg_outcome.can_handshake_v4 = response.can_handshake; + wg_outcome.can_resolve_dns_v4 = response.can_resolve_dns; + wg_outcome.ping_hosts_performance_v4 = + safe_ratio(response.received_hosts, response.sent_hosts); + wg_outcome.ping_ips_performance_v4 = + safe_ratio(response.received_ips, response.sent_ips); + + wg_outcome.download_duration_sec_v4 = response.download_duration_sec; + wg_outcome.download_duration_milliseconds_v4 = response.download_duration_milliseconds; + wg_outcome.downloaded_file_size_bytes_v4 = response.downloaded_file_size_bytes; + wg_outcome.downloaded_file_v4 = response.downloaded_file; + wg_outcome.download_error_v4 = response.download_error; + } + Ok(NetstackResult::Error { error }) => { + error!("Two-hop netstack runtime error (IPv4): {error}") + } + Err(error) => { + error!("Two-hop internal error (IPv4): {error}") + } + } +} diff --git a/nym-gateway-probe/src/lib.rs b/nym-gateway-probe/src/lib.rs index d2413be4254..e25f283889f 100644 --- a/nym-gateway-probe/src/lib.rs +++ b/nym-gateway-probe/src/lib.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use crate::{netstack::NetstackResult, types::Entry}; +use crate::types::Entry; use anyhow::bail; use base64::{Engine as _, engine::general_purpose}; use bytes::BytesMut; @@ -20,7 +20,7 @@ use nym_authenticator_requests::{ }; use nym_client_core::config::ForgetMe; use nym_config::defaults::{ - NymNetworkDetails, WG_METADATA_PORT, WG_TUN_DEVICE_IP_ADDRESS_V4, + NymNetworkDetails, mixnet_vpn::{NYM_TUN_DEVICE_ADDRESS_V4, NYM_TUN_DEVICE_ADDRESS_V6}, }; use nym_connection_monitor::self_ping_and_wait; @@ -51,16 +51,17 @@ use crate::{ types::Exit, }; -use netstack::{NetstackRequest, NetstackRequestGo}; - mod bandwidth_helpers; +mod common; mod icmp; +pub mod mode; mod netstack; pub mod nodes; mod types; use crate::bandwidth_helpers::{acquire_bandwidth, import_bandwidth}; use crate::nodes::{DirectoryNode, NymApiDirectory}; +pub use mode::TestMode; use nym_node_status_client::models::AttachedTicketMaterials; pub use types::{IpPingReplies, ProbeOutcome, ProbeResult}; @@ -149,6 +150,32 @@ pub struct TestedNodeDetails { lp_address: Option, } +impl TestedNodeDetails { + /// Create from CLI args (localnet mode - no HTTP query needed) + /// Only identity and LP address are required; other fields are None/default. + pub fn from_cli(identity: NodeIdentity, lp_address: std::net::SocketAddr) -> Self { + Self { + identity, + ip_address: Some(lp_address.ip()), + lp_address: Some(lp_address), + // These are None in localnet mode - only needed for mixnet/authenticator + exit_router_address: None, + authenticator_address: None, + authenticator_version: AuthenticatorVersion::UNKNOWN, + } + } + + /// Check if this node has sufficient info for LP testing + pub fn can_test_lp(&self) -> bool { + self.lp_address.is_some() + } + + /// Check if this node has sufficient info for mixnet testing + pub fn can_test_mixnet(&self) -> bool { + self.exit_router_address.is_some() || self.authenticator_address.is_some() + } +} + pub struct Probe { entrypoint: NodeIdentity, tested_node: TestedNode, @@ -159,6 +186,10 @@ pub struct Probe { direct_gateway_node: Option, /// Pre-queried exit gateway node (used when --exit-gateway-ip is specified for LP forwarding) exit_gateway_node: Option, + /// Localnet entry gateway info (used when --entry-gateway-identity is specified) + localnet_entry: Option, + /// Localnet exit gateway info (used when --exit-gateway-identity is specified) + localnet_exit: Option, } impl Probe { @@ -176,6 +207,8 @@ impl Probe { credentials_args, direct_gateway_node: None, exit_gateway_node: None, + localnet_entry: None, + localnet_exit: None, } } @@ -195,6 +228,8 @@ impl Probe { credentials_args, direct_gateway_node: Some(gateway_node), exit_gateway_node: None, + localnet_entry: None, + localnet_exit: None, } } @@ -215,6 +250,30 @@ impl Probe { credentials_args, direct_gateway_node: Some(entry_gateway_node), exit_gateway_node: Some(exit_gateway_node), + localnet_entry: None, + localnet_exit: None, + } + } + + /// Create a probe for localnet mode (no HTTP query needed) + /// Uses identity + LP address directly from CLI args + pub fn new_localnet( + entry: TestedNodeDetails, + exit: Option, + netstack_args: NetstackArgs, + credentials_args: CredentialArgs, + ) -> Self { + let entrypoint = entry.identity; + Self { + entrypoint, + tested_node: TestedNode::SameAsEntry, + amnezia_args: "".into(), + netstack_args, + credentials_args, + direct_gateway_node: None, + exit_gateway_node: None, + localnet_entry: Some(entry), + localnet_exit: exit, } } @@ -223,6 +282,7 @@ impl Probe { self } + #[allow(clippy::too_many_arguments)] pub async fn probe( self, directory: Option, @@ -258,6 +318,11 @@ impl Probe { let mixnet_client = Box::pin(disconnected_mixnet_client.connect_to_mixnet()).await; + // Convert legacy flags to TestMode + let has_exit = self.exit_gateway_node.is_some() || self.localnet_exit.is_some(); + let test_mode = + TestMode::from_flags(only_wireguard, only_lp_registration, test_lp_wg, has_exit); + self.do_probe_test( Some(mixnet_client), storage, @@ -266,9 +331,8 @@ impl Probe { directory.as_ref(), nyxd_url, tested_entry, + test_mode, only_wireguard, - only_lp_registration, - test_lp_wg, false, // Not using mock ecash in regular probe mode ) .await @@ -288,11 +352,62 @@ impl Probe { min_mixnet_performance: Option, use_mock_ecash: bool, ) -> anyhow::Result { + // Localnet mode - identity + LP address from CLI, no HTTP query + // This path is used when --entry-gateway-identity is specified + if let Some(entry_info) = &self.localnet_entry { + info!("Using localnet mode with CLI-provided gateway identities"); + + // Initialize storage (needed for credentials) + if !config_dir.exists() { + std::fs::create_dir_all(config_dir)?; + } + let storage_paths = StoragePaths::new_from_dir(config_dir)?; + let storage = storage_paths + .initialise_default_persistent_storage() + .await?; + + // For localnet, use entry as the test node (or exit if provided) + let mixnet_entry_gateway_id = entry_info.identity; + let node_info = if let Some(exit_info) = &self.localnet_exit { + exit_info.clone() + } else { + entry_info.clone() + }; + + // Convert legacy flags to TestMode + let has_exit = self.localnet_exit.is_some(); + let test_mode = + TestMode::from_flags(only_wireguard, only_lp_registration, test_lp_wg, has_exit); + + return self + .do_probe_test( + None, + storage, + mixnet_entry_gateway_id, + node_info, + directory.as_ref(), + nyxd_url, + false, // tested_entry + test_mode, + only_wireguard, + use_mock_ecash, + ) + .await; + } + // If both gateways are pre-queried via --gateway-ip and --exit-gateway-ip, // skip mixnet setup entirely - we have all the data we need if self.direct_gateway_node.is_some() && self.exit_gateway_node.is_some() { - let entry_node = self.direct_gateway_node.as_ref().unwrap(); - let exit_node = self.exit_gateway_node.as_ref().unwrap(); + let entry_node = if let Some(entry_node) = self.direct_gateway_node.as_ref() { + entry_node + } else { + return Err(anyhow::anyhow!("Entry gateway node is missing")); + }; + let exit_node = if let Some(exit_node) = self.exit_gateway_node.as_ref() { + exit_node + } else { + return Err(anyhow::anyhow!("Exit gateway node is missing")); + }; // Initialize storage (needed for credentials) if !config_dir.exists() { @@ -307,6 +422,10 @@ impl Probe { let mixnet_entry_gateway_id = entry_node.identity(); let node_info = exit_node.to_testable_node()?; + // Convert legacy flags to TestMode (has_exit = true since we have exit_gateway_node) + let test_mode = + TestMode::from_flags(only_wireguard, only_lp_registration, test_lp_wg, true); + return self .do_probe_test( None, @@ -316,9 +435,8 @@ impl Probe { directory.as_ref(), nyxd_url, false, // tested_entry + test_mode, only_wireguard, - only_lp_registration, - test_lp_wg, use_mock_ecash, ) .await; @@ -395,6 +513,11 @@ impl Probe { let mixnet_client = Box::pin(disconnected_mixnet_client.connect_to_mixnet()).await; + // Convert legacy flags to TestMode + let has_exit = self.exit_gateway_node.is_some() || self.localnet_exit.is_some(); + let test_mode = + TestMode::from_flags(only_wireguard, only_lp_registration, test_lp_wg, has_exit); + self.do_probe_test( Some(mixnet_client), storage, @@ -403,9 +526,8 @@ impl Probe { directory.as_ref(), nyxd_url, tested_entry, + test_mode, only_wireguard, - only_lp_registration, - test_lp_wg, use_mock_ecash, ) .await @@ -549,15 +671,16 @@ impl Probe { directory: Option<&NymApiDirectory>, nyxd_url: Url, tested_entry: bool, + test_mode: TestMode, only_wireguard: bool, - only_lp_registration: bool, - test_lp_wg: bool, use_mock_ecash: bool, ) -> anyhow::Result where T: MixnetClientStorage + Clone + 'static, ::StorageError: Send + Sync, { + // test_mode replaces the old only_lp_registration and test_lp_wg flags. + // only_wireguard is kept separate as it controls ping behavior within Mixnet mode. let mut rng = rand::thread_rng(); let mixnet_client = match mixnet_client { Some(Ok(mixnet_client)) => Some(mixnet_client), @@ -581,6 +704,11 @@ impl Probe { None => None, }; + // Determine if we should run ping tests: + // - Only in Mixnet mode (LP modes don't use mixnet) + // - And only if not --only-wireguard (which skips pings) + let run_ping_tests = test_mode.needs_mixnet() && !only_wireguard; + let (outcome, mixnet_client) = if let Some(mixnet_client) = mixnet_client { let nym_address = *mixnet_client.nym_address(); let entry_gateway = nym_address.gateway().to_base58_string(); @@ -588,8 +716,16 @@ impl Probe { info!("Successfully connected to entry gateway: {entry_gateway}"); info!("Our nym address: {nym_address}"); - // Now that we have a connected mixnet client, we can start pinging - let (outcome, mixnet_client) = if only_wireguard || only_lp_registration { + // Run ping tests if applicable + let (outcome, mixnet_client) = if run_ping_tests { + do_ping( + mixnet_client, + nym_address, + node_info.exit_router_address, + tested_entry, + ) + .await + } else { ( Ok(ProbeOutcome { as_entry: if tested_entry { @@ -603,43 +739,41 @@ impl Probe { }), mixnet_client, ) - } else { - do_ping( - mixnet_client, - nym_address, - node_info.exit_router_address, - tested_entry, - ) - .await }; (outcome, Some(mixnet_client)) - } else if test_lp_wg { - // No mixnet client needed for LP-WG test with pre-queried nodes + } else if test_mode.uses_lp() && test_mode.tests_wireguard() { + // LP modes (SingleHop/TwoHop) don't need mixnet client // Create default outcome and continue to LP-WG test below - (Ok(ProbeOutcome { - as_entry: Entry::NotTested, - as_exit: None, - wg: None, - lp: None, - }), None) + ( + Ok(ProbeOutcome { + as_entry: Entry::NotTested, + as_exit: None, + wg: None, + lp: None, + }), + None, + ) } else { - // For non-LP-WG modes, missing mixnet client is a failure - (Ok(ProbeOutcome { - as_entry: if tested_entry { - Entry::fail_to_connect() - } else { - Entry::EntryFailure - }, - as_exit: None, - wg: None, - lp: None, - }), None) + // For Mixnet mode, missing mixnet client is a failure + ( + Ok(ProbeOutcome { + as_entry: if tested_entry { + Entry::fail_to_connect() + } else { + Entry::EntryFailure + }, + as_exit: None, + wg: None, + lp: None, + }), + None, + ) }; - let wg_outcome = if only_lp_registration { - // Skip WireGuard test when only testing LP registration + let wg_outcome = if !test_mode.tests_wireguard() { + // LpOnly mode: skip WireGuard test WgProbeResults::default() - } else if test_lp_wg { + } else if test_mode.uses_lp() { // Test WireGuard via LP registration (nested session forwarding) info!("Testing WireGuard via LP registration (no mixnet)"); @@ -655,6 +789,10 @@ impl Probe { ); // Determine entry and exit gateways + // Three modes for gateway resolution: + // 1. direct_gateway_node/exit_gateway_node - from --gateway-ip (HTTP API query) + // 2. localnet_entry/localnet_exit - from --entry-gateway-identity (CLI-only) + // 3. directory lookup - original behavior for production let (entry_gateway, exit_gateway) = if let Some(exit_node) = &self.exit_gateway_node { // Both entry and exit gateways were pre-queried (direct IP mode) info!("Using pre-queried entry and exit gateways for LP forwarding test"); @@ -667,6 +805,14 @@ impl Probe { let exit_gateway = exit_node.to_testable_node()?; (entry_gateway, exit_gateway) + } else if let Some(exit_localnet) = &self.localnet_exit { + // Localnet mode: use CLI-provided identities and LP addresses + info!("Using localnet entry and exit gateways for LP forwarding test"); + let entry_localnet = self.localnet_entry.as_ref().ok_or_else(|| { + anyhow::anyhow!("Entry gateway not available in localnet mode") + })?; + + (entry_localnet.clone(), exit_localnet.clone()) } else { // Original behavior: query from directory // The tested node is the exit @@ -684,7 +830,6 @@ impl Probe { &entry_gateway, &exit_gateway, &bw_controller, - storage.credential_store().clone(), use_mock_ecash, self.amnezia_args.clone(), self.netstack_args.clone(), @@ -895,84 +1040,16 @@ async fn wg_probe( wg_outcome.can_register = true; - if wg_outcome.can_register { - let netstack_request = NetstackRequest::new( - ®istered_data.private_ips().ipv4.to_string(), - ®istered_data.private_ips().ipv6.to_string(), - &private_key_hex, - &public_key_hex, - &wg_endpoint, - &format!("http://{WG_TUN_DEVICE_IP_ADDRESS_V4}:{WG_METADATA_PORT}"), - netstack_args.netstack_download_timeout_sec, - &awg_args, - netstack_args, - ); - - // Perform IPv4 ping test - let ipv4_request = NetstackRequestGo::from_rust_v4(&netstack_request); - - match netstack::ping(&ipv4_request) { - Ok(NetstackResult::Response(netstack_response_v4)) => { - info!( - "Wireguard probe response for IPv4: {:#?}", - netstack_response_v4 - ); - wg_outcome.can_query_metadata_v4 = netstack_response_v4.can_query_metadata; - wg_outcome.can_handshake_v4 = netstack_response_v4.can_handshake; - wg_outcome.can_resolve_dns_v4 = netstack_response_v4.can_resolve_dns; - wg_outcome.ping_hosts_performance_v4 = netstack_response_v4.received_hosts as f32 - / netstack_response_v4.sent_hosts as f32; - wg_outcome.ping_ips_performance_v4 = - netstack_response_v4.received_ips as f32 / netstack_response_v4.sent_ips as f32; - - wg_outcome.download_duration_sec_v4 = netstack_response_v4.download_duration_sec; - wg_outcome.download_duration_milliseconds_v4 = - netstack_response_v4.download_duration_milliseconds; - wg_outcome.downloaded_file_size_bytes_v4 = - netstack_response_v4.downloaded_file_size_bytes; - wg_outcome.downloaded_file_v4 = netstack_response_v4.downloaded_file; - wg_outcome.download_error_v4 = netstack_response_v4.download_error; - } - Ok(NetstackResult::Error { error }) => { - error!("Netstack runtime error: {error}") - } - Err(error) => { - error!("Internal error: {error}") - } - } - - // Perform IPv6 ping test - let ipv6_request = NetstackRequestGo::from_rust_v6(&netstack_request); + // Run tunnel connectivity tests using shared helper + let tunnel_config = common::WgTunnelConfig::new( + registered_data.private_ips().ipv4.to_string(), + registered_data.private_ips().ipv6.to_string(), + private_key_hex, + public_key_hex, + wg_endpoint, + ); - match netstack::ping(&ipv6_request) { - Ok(NetstackResult::Response(netstack_response_v6)) => { - info!( - "Wireguard probe response for IPv6: {:#?}", - netstack_response_v6 - ); - wg_outcome.can_handshake_v6 = netstack_response_v6.can_handshake; - wg_outcome.can_resolve_dns_v6 = netstack_response_v6.can_resolve_dns; - wg_outcome.ping_hosts_performance_v6 = netstack_response_v6.received_hosts as f32 - / netstack_response_v6.sent_hosts as f32; - wg_outcome.ping_ips_performance_v6 = - netstack_response_v6.received_ips as f32 / netstack_response_v6.sent_ips as f32; - - wg_outcome.download_duration_sec_v6 = netstack_response_v6.download_duration_sec; - wg_outcome.download_duration_milliseconds_v6 = - netstack_response_v6.download_duration_milliseconds; - wg_outcome.downloaded_file_size_bytes_v6 = - netstack_response_v6.downloaded_file_size_bytes; - wg_outcome.downloaded_file_v6 = netstack_response_v6.downloaded_file; - wg_outcome.download_error_v6 = netstack_response_v6.download_error; - } - Ok(NetstackResult::Error { error }) => { - error!("Netstack runtime error: {error}") - } - Err(error) => { - error!("Internal error: {error}") - } - } - } + common::run_tunnel_tests(&tunnel_config, &netstack_args, &awg_args, &mut wg_outcome); Ok(wg_outcome) } @@ -1013,26 +1090,14 @@ where gateway_ip, ); - // Step 1: Connect to gateway - info!("Connecting to LP listener at {}...", gateway_lp_address); - match client.connect().await { - Ok(_) => { - info!("Successfully connected to LP listener"); - lp_outcome.can_connect = true; - } - Err(e) => { - let error_msg = format!("Failed to connect to LP listener: {}", e); - error!("{}", error_msg); - lp_outcome.error = Some(error_msg); - return Ok(lp_outcome); - } - } - - // Step 2: Perform handshake - info!("Performing LP handshake..."); + // Step 1: Perform handshake (connection is implicit in packet-per-connection model) + // LpRegistrationClient uses packet-per-connection model - connect() is gone, + // connection is established during handshake and registration automatically. + info!("Performing LP handshake at {}...", gateway_lp_address); match client.perform_handshake().await { Ok(_) => { info!("LP handshake completed successfully"); + lp_outcome.can_connect = true; // Connection succeeded if handshake succeeded lp_outcome.can_handshake = true; } Err(e) => { @@ -1043,7 +1108,7 @@ where } } - // Step 3: Send registration request + // Step 2: Register with gateway (send request + receive response in one call) info!("Sending LP registration request..."); // Generate WireGuard keypair for dVPN registration @@ -1063,9 +1128,9 @@ where } }; - // Generate credential based on mode + // Register using the new packet-per-connection API (returns GatewayData directly) let ticket_type = TicketType::V1WireguardEntry; - if use_mock_ecash { + let gateway_data = if use_mock_ecash { info!("Using mock ecash credential for LP registration"); let credential = crate::bandwidth_helpers::create_dummy_credential( &gateway_ed25519_pubkey.to_bytes(), @@ -1073,19 +1138,12 @@ where ); match client - .send_registration_request_with_credential( - &wg_keypair, - &gateway_ed25519_pubkey, - credential, - ticket_type, - ) + .register_with_credential(&wg_keypair, credential, ticket_type) .await { - Ok(_) => { - info!("LP registration request sent successfully with mock ecash"); - } + Ok(data) => data, Err(e) => { - let error_msg = format!("Failed to send LP registration request: {}", e); + let error_msg = format!("LP registration failed (mock ecash): {}", e); error!("{}", error_msg); lp_outcome.error = Some(error_msg); return Ok(lp_outcome); @@ -1094,7 +1152,7 @@ where } else { info!("Using real bandwidth controller for LP registration"); match client - .send_registration_request( + .register( &wg_keypair, &gateway_ed25519_pubkey, bandwidth_controller, @@ -1102,36 +1160,22 @@ where ) .await { - Ok(_) => { - info!("LP registration request sent successfully with real ecash"); - } + Ok(data) => data, Err(e) => { - let error_msg = format!("Failed to send LP registration request: {}", e); + let error_msg = format!("LP registration failed: {}", e); error!("{}", error_msg); lp_outcome.error = Some(error_msg); return Ok(lp_outcome); } } - } + }; - // Step 4: Receive registration response - info!("Waiting for LP registration response..."); - match client.receive_registration_response().await { - Ok(gateway_data) => { - info!("LP registration successful! Received gateway data:"); - info!(" - Gateway public key: {:?}", gateway_data.public_key); - info!(" - Private IPv4: {}", gateway_data.private_ipv4); - info!(" - Private IPv6: {}", gateway_data.private_ipv6); - info!(" - Endpoint: {}", gateway_data.endpoint); - lp_outcome.can_register = true; - } - Err(e) => { - let error_msg = format!("Failed to receive LP registration response: {}", e); - error!("{}", error_msg); - lp_outcome.error = Some(error_msg); - return Ok(lp_outcome); - } - } + info!("LP registration successful! Received gateway data:"); + info!(" - Gateway public key: {:?}", gateway_data.public_key); + info!(" - Private IPv4: {}", gateway_data.private_ipv4); + info!(" - Private IPv6: {}", gateway_data.private_ipv6); + info!(" - Endpoint: {}", gateway_data.endpoint); + lp_outcome.can_register = true; Ok(lp_outcome) } @@ -1146,6 +1190,13 @@ where /// /// This validates that IP hiding works (exit sees entry IP, not client IP) and that the /// full VPN tunnel operates correctly after LP registration. +/// +// Known issue in localnet mode - After this probe runs, container networking +// to the external internet becomes unstable while internal container-to-container traffic +// continues to work. The two-hop WireGuard tunnel itself succeeds (handshake completes), +// but subsequent DNS/ping tests may timeout. This appears to be related to Apple Container +// Runtime networking quirks combined with our NAT/iptables configuration. Tracked in +// beads issue nym-vbdo. Workaround: restart the localnet containers between probe runs. async fn wg_probe_lp( entry_gateway: &TestedNodeDetails, exit_gateway: &TestedNodeDetails, @@ -1153,7 +1204,6 @@ async fn wg_probe_lp( nym_validator_client::nyxd::NyxdClient, St, >, - _storage: St, use_mock_ecash: bool, awg_args: String, netstack_args: NetstackArgs, @@ -1193,7 +1243,9 @@ where let exit_wg_keypair = x25519::KeyPair::new(&mut rng); // STEP 1: Establish outer LP session with entry gateway - info!("Connecting to entry gateway via LP..."); + // LpRegistrationClient uses packet-per-connection model - connect() is gone, + // connection is established automatically during handshake. + info!("Establishing outer LP session with entry gateway..."); let mut entry_client = LpRegistrationClient::new_with_default_psk( entry_lp_keypair, entry_gateway.identity, @@ -1201,13 +1253,7 @@ where entry_ip, ); - // Connect to entry gateway - if let Err(e) = entry_client.connect().await { - error!("Failed to connect to entry gateway: {}", e); - return Ok(wg_outcome); - } - - // Perform handshake with entry gateway + // Perform handshake with entry gateway (connection is implicit) if let Err(e) = entry_client.perform_handshake().await { error!("Failed to handshake with entry gateway: {}", e); return Ok(wg_outcome); @@ -1229,24 +1275,45 @@ where .map_err(|e| anyhow::anyhow!("Invalid exit gateway identity: {}", e))?; // Perform handshake and registration with exit gateway via forwarding - if use_mock_ecash { - info!("Note: Using mock ecash mode - gateways must be started with --lp-use-mock-ecash"); - } - let exit_gateway_data = match nested_session - .handshake_and_register( - &mut entry_client, - &exit_wg_keypair, - &exit_gateway_pubkey, - bandwidth_controller, + let exit_gateway_data = if use_mock_ecash { + info!("Using mock ecash credential for exit gateway registration"); + let credential = crate::bandwidth_helpers::create_dummy_credential( + &exit_gateway_pubkey.to_bytes(), TicketType::V1WireguardExit, - exit_ip, - ) - .await - { - Ok(data) => data, - Err(e) => { - error!("Failed to register with exit gateway: {}", e); - return Ok(wg_outcome); + ); + match nested_session + .handshake_and_register_with_credential( + &mut entry_client, + &exit_wg_keypair, + credential, + TicketType::V1WireguardExit, + exit_ip, + ) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with exit gateway (mock ecash): {}", e); + return Ok(wg_outcome); + } + } + } else { + match nested_session + .handshake_and_register( + &mut entry_client, + &exit_wg_keypair, + &exit_gateway_pubkey, + bandwidth_controller, + TicketType::V1WireguardExit, + exit_ip, + ) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with exit gateway: {}", e); + return Ok(wg_outcome); + } } }; info!("Exit gateway registration successful via forwarding"); @@ -1257,24 +1324,38 @@ where ed25519::PublicKey::from_bytes(&entry_gateway.identity.to_bytes()) .map_err(|e| anyhow::anyhow!("Invalid entry gateway identity: {}", e))?; - if let Err(e) = entry_client - .send_registration_request( - &entry_wg_keypair, - &entry_gateway_pubkey, - bandwidth_controller, + // Use packet-per-connection register() which returns GatewayData directly + let entry_gateway_data = if use_mock_ecash { + info!("Using mock ecash credential for entry gateway registration"); + let credential = crate::bandwidth_helpers::create_dummy_credential( + &entry_gateway_pubkey.to_bytes(), TicketType::V1WireguardEntry, - ) - .await - { - error!("Failed to send entry registration request: {}", e); - return Ok(wg_outcome); - } - - let _entry_gateway_data = match entry_client.receive_registration_response().await { - Ok(data) => data, - Err(e) => { - error!("Failed to receive entry registration response: {}", e); - return Ok(wg_outcome); + ); + match entry_client + .register_with_credential(&entry_wg_keypair, credential, TicketType::V1WireguardEntry) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with entry gateway (mock ecash): {}", e); + return Ok(wg_outcome); + } + } + } else { + match entry_client + .register( + &entry_wg_keypair, + &entry_gateway_pubkey, + bandwidth_controller, + TicketType::V1WireguardEntry, + ) + .await + { + Ok(data) => data, + Err(e) => { + error!("Failed to register with entry gateway: {}", e); + return Ok(wg_outcome); + } } }; info!("Entry gateway registration successful"); @@ -1282,100 +1363,49 @@ where info!("LP registration successful for both gateways!"); wg_outcome.can_register = true; - // STEP 4: Test WireGuard tunnels using exit gateway configuration + // STEP 4: Test WireGuard tunnels using two-hop configuration + // Traffic flows: Exit tunnel -> UDP Forwarder -> Entry tunnel -> Exit Gateway -> Internet + // The exit gateway endpoint is not directly reachable from the host in localnet. + // We must tunnel through the entry gateway using the UDP forwarder pattern. + // Convert keys to hex for netstack - let private_key_hex = hex::encode(exit_wg_keypair.private_key().to_bytes()); - let public_key_hex = hex::encode(exit_gateway_data.public_key.to_bytes()); - - // Build WireGuard endpoint address - let wg_endpoint = format!("{}:{}", exit_ip, exit_gateway_data.endpoint.port()); - - info!("Exit WireGuard configuration:"); - info!(" Private IPv4: {}", exit_gateway_data.private_ipv4); - info!(" Private IPv6: {}", exit_gateway_data.private_ipv6); - info!(" Endpoint: {}", wg_endpoint); - - // Run tunnel tests (copied from wg_probe) - let netstack_request = crate::netstack::NetstackRequest::new( - &exit_gateway_data.private_ipv4.to_string(), - &exit_gateway_data.private_ipv6.to_string(), - &private_key_hex, - &public_key_hex, - &wg_endpoint, - &format!("http://{WG_TUN_DEVICE_IP_ADDRESS_V4}:{WG_METADATA_PORT}"), - netstack_args.netstack_download_timeout_sec, - &awg_args, - netstack_args, + let entry_private_key_hex = hex::encode(entry_wg_keypair.private_key().to_bytes()); + let entry_public_key_hex = hex::encode(entry_gateway_data.public_key.to_bytes()); + let exit_private_key_hex = hex::encode(exit_wg_keypair.private_key().to_bytes()); + let exit_public_key_hex = hex::encode(exit_gateway_data.public_key.to_bytes()); + + // Build WireGuard endpoint addresses + // Entry endpoint uses entry_ip (host-reachable) + port from registration + let entry_wg_endpoint = format!("{}:{}", entry_ip, entry_gateway_data.endpoint.port()); + // Exit endpoint uses exit_ip + port from registration (forwarded via entry) + let exit_wg_endpoint = format!("{}:{}", exit_ip, exit_gateway_data.endpoint.port()); + + info!("Two-hop WireGuard configuration:"); + info!(" Entry gateway:"); + info!(" Private IPv4: {}", entry_gateway_data.private_ipv4); + info!(" Endpoint: {}", entry_wg_endpoint); + info!(" Exit gateway:"); + info!(" Private IPv4: {}", exit_gateway_data.private_ipv4); + info!(" Endpoint (via forwarder): {}", exit_wg_endpoint); + + // Build two-hop tunnel configuration + let two_hop_config = common::TwoHopWgTunnelConfig::new( + entry_gateway_data.private_ipv4.to_string(), + entry_private_key_hex, + entry_public_key_hex, + entry_wg_endpoint, + awg_args.clone(), // Entry AWG args + exit_gateway_data.private_ipv4.to_string(), + exit_private_key_hex, + exit_public_key_hex, + exit_wg_endpoint, + awg_args, // Exit AWG args ); - // Perform IPv4 ping test - info!("Testing IPv4 tunnel connectivity..."); - let ipv4_request = crate::netstack::NetstackRequestGo::from_rust_v4(&netstack_request); - - match crate::netstack::ping(&ipv4_request) { - Ok(NetstackResult::Response(netstack_response_v4)) => { - info!( - "Wireguard probe response for IPv4: {:#?}", - netstack_response_v4 - ); - wg_outcome.can_query_metadata_v4 = netstack_response_v4.can_query_metadata; - wg_outcome.can_handshake_v4 = netstack_response_v4.can_handshake; - wg_outcome.can_resolve_dns_v4 = netstack_response_v4.can_resolve_dns; - wg_outcome.ping_hosts_performance_v4 = - netstack_response_v4.received_hosts as f32 / netstack_response_v4.sent_hosts as f32; - wg_outcome.ping_ips_performance_v4 = - netstack_response_v4.received_ips as f32 / netstack_response_v4.sent_ips as f32; - - wg_outcome.download_duration_sec_v4 = netstack_response_v4.download_duration_sec; - wg_outcome.download_duration_milliseconds_v4 = - netstack_response_v4.download_duration_milliseconds; - wg_outcome.downloaded_file_size_bytes_v4 = - netstack_response_v4.downloaded_file_size_bytes; - wg_outcome.downloaded_file_v4 = netstack_response_v4.downloaded_file; - wg_outcome.download_error_v4 = netstack_response_v4.download_error; - } - Ok(NetstackResult::Error { error }) => { - error!("Netstack runtime error (IPv4): {error}") - } - Err(error) => { - error!("Internal error (IPv4): {error}") - } - } - - // Perform IPv6 ping test - info!("Testing IPv6 tunnel connectivity..."); - let ipv6_request = crate::netstack::NetstackRequestGo::from_rust_v6(&netstack_request); - - match crate::netstack::ping(&ipv6_request) { - Ok(NetstackResult::Response(netstack_response_v6)) => { - info!( - "Wireguard probe response for IPv6: {:#?}", - netstack_response_v6 - ); - wg_outcome.can_handshake_v6 = netstack_response_v6.can_handshake; - wg_outcome.can_resolve_dns_v6 = netstack_response_v6.can_resolve_dns; - wg_outcome.ping_hosts_performance_v6 = - netstack_response_v6.received_hosts as f32 / netstack_response_v6.sent_hosts as f32; - wg_outcome.ping_ips_performance_v6 = - netstack_response_v6.received_ips as f32 / netstack_response_v6.sent_ips as f32; - - wg_outcome.download_duration_sec_v6 = netstack_response_v6.download_duration_sec; - wg_outcome.download_duration_milliseconds_v6 = - netstack_response_v6.download_duration_milliseconds; - wg_outcome.downloaded_file_size_bytes_v6 = - netstack_response_v6.downloaded_file_size_bytes; - wg_outcome.downloaded_file_v6 = netstack_response_v6.downloaded_file; - wg_outcome.download_error_v6 = netstack_response_v6.download_error; - } - Ok(NetstackResult::Error { error }) => { - error!("Netstack runtime error (IPv6): {error}") - } - Err(error) => { - error!("Internal error (IPv6): {error}") - } - } + // Run two-hop tunnel connectivity tests + common::run_two_hop_tunnel_tests(&two_hop_config, &netstack_args, &mut wg_outcome); - info!("LP-based WireGuard probe completed"); + info!("LP-based two-hop WireGuard probe completed"); Ok(wg_outcome) } diff --git a/nym-gateway-probe/src/mode/mod.rs b/nym-gateway-probe/src/mode/mod.rs new file mode 100644 index 00000000000..5c6e45450d3 --- /dev/null +++ b/nym-gateway-probe/src/mode/mod.rs @@ -0,0 +1,283 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +//! Test mode definitions for gateway probe. +//! +//! This module defines the different test modes supported by the gateway probe: +//! - Mixnet: Traditional mixnet path testing +//! - SingleHop: LP registration + WireGuard on single gateway +//! - TwoHop: Entry LP + Exit LP (nested forwarding) + WireGuard +//! - LpOnly: LP registration only, no WireGuard + +/// Test mode for the gateway probe. +/// +/// Determines which tests are performed and how connections are established. +// This enum replaces the scattered boolean flags (only_wireguard, +// only_lp_registration, test_lp_wg) with explicit, named modes for clarity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TestMode { + /// Traditional mixnet testing - connects via mixnet, tests entry/exit pings + WireGuard via authenticator + #[default] + Mixnet, + /// LP registration + WireGuard on single gateway (no mixnet, no forwarding) + SingleHop, + /// Entry LP + Exit LP (nested session forwarding) + WireGuard tunnel + TwoHop, + /// LP registration only - test handshake and registration, skip WireGuard + LpOnly, +} + +impl TestMode { + /// Infer test mode from legacy boolean flags (backward compatibility) + pub fn from_flags( + only_wireguard: bool, + only_lp_registration: bool, + test_lp_wg: bool, + has_exit_gateway: bool, + ) -> Self { + if only_lp_registration { + TestMode::LpOnly + } else if test_lp_wg { + if has_exit_gateway { + TestMode::TwoHop + } else { + TestMode::SingleHop + } + } else if only_wireguard { + // WireGuard via authenticator (still uses mixnet path) + TestMode::Mixnet + } else { + TestMode::Mixnet + } + } + + /// Whether this mode requires a mixnet client + pub fn needs_mixnet(&self) -> bool { + matches!(self, TestMode::Mixnet) + } + + /// Whether this mode uses LP registration + pub fn uses_lp(&self) -> bool { + matches!( + self, + TestMode::SingleHop | TestMode::TwoHop | TestMode::LpOnly + ) + } + + /// Whether this mode tests WireGuard tunnels + pub fn tests_wireguard(&self) -> bool { + matches!( + self, + TestMode::Mixnet | TestMode::SingleHop | TestMode::TwoHop + ) + } + + /// Whether this mode requires an exit gateway + pub fn needs_exit_gateway(&self) -> bool { + matches!(self, TestMode::TwoHop) + } +} + +impl std::fmt::Display for TestMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TestMode::Mixnet => write!(f, "mixnet"), + TestMode::SingleHop => write!(f, "single-hop"), + TestMode::TwoHop => write!(f, "two-hop"), + TestMode::LpOnly => write!(f, "lp-only"), + } + } +} + +impl std::str::FromStr for TestMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "mixnet" => Ok(TestMode::Mixnet), + "single-hop" | "singlehop" | "single_hop" => Ok(TestMode::SingleHop), + "two-hop" | "twohop" | "two_hop" => Ok(TestMode::TwoHop), + "lp-only" | "lponly" | "lp_only" => Ok(TestMode::LpOnly), + _ => Err(format!( + "Unknown test mode: '{}'. Valid modes: mixnet, single-hop, two-hop, lp-only", + s + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ============ from_flags() tests ============ + + #[test] + fn test_from_flags_default_is_mixnet() { + // All flags false -> Mixnet (default) + assert_eq!( + TestMode::from_flags(false, false, false, false), + TestMode::Mixnet + ); + } + + #[test] + fn test_from_flags_only_wireguard_is_mixnet() { + // only_wireguard still uses mixnet path (WG via authenticator) + assert_eq!( + TestMode::from_flags(true, false, false, false), + TestMode::Mixnet + ); + } + + #[test] + fn test_from_flags_only_lp_registration() { + // only_lp_registration -> LpOnly (takes priority) + assert_eq!( + TestMode::from_flags(false, true, false, false), + TestMode::LpOnly + ); + // Even with other flags set, only_lp_registration wins + assert_eq!( + TestMode::from_flags(true, true, true, true), + TestMode::LpOnly + ); + } + + #[test] + fn test_from_flags_test_lp_wg_single_hop() { + // test_lp_wg without exit gateway -> SingleHop + assert_eq!( + TestMode::from_flags(false, false, true, false), + TestMode::SingleHop + ); + } + + #[test] + fn test_from_flags_test_lp_wg_two_hop() { + // test_lp_wg with exit gateway -> TwoHop + assert_eq!( + TestMode::from_flags(false, false, true, true), + TestMode::TwoHop + ); + } + + #[test] + fn test_from_flags_has_exit_gateway_alone_is_mixnet() { + // has_exit_gateway alone doesn't change mode + assert_eq!( + TestMode::from_flags(false, false, false, true), + TestMode::Mixnet + ); + } + + // ============ Helper method tests ============ + + #[test] + fn test_needs_mixnet() { + assert!(TestMode::Mixnet.needs_mixnet()); + assert!(!TestMode::SingleHop.needs_mixnet()); + assert!(!TestMode::TwoHop.needs_mixnet()); + assert!(!TestMode::LpOnly.needs_mixnet()); + } + + #[test] + fn test_uses_lp() { + assert!(!TestMode::Mixnet.uses_lp()); + assert!(TestMode::SingleHop.uses_lp()); + assert!(TestMode::TwoHop.uses_lp()); + assert!(TestMode::LpOnly.uses_lp()); + } + + #[test] + fn test_tests_wireguard() { + assert!(TestMode::Mixnet.tests_wireguard()); + assert!(TestMode::SingleHop.tests_wireguard()); + assert!(TestMode::TwoHop.tests_wireguard()); + assert!(!TestMode::LpOnly.tests_wireguard()); + } + + #[test] + fn test_needs_exit_gateway() { + assert!(!TestMode::Mixnet.needs_exit_gateway()); + assert!(!TestMode::SingleHop.needs_exit_gateway()); + assert!(TestMode::TwoHop.needs_exit_gateway()); + assert!(!TestMode::LpOnly.needs_exit_gateway()); + } + + // ============ Display tests ============ + + #[test] + fn test_display() { + assert_eq!(TestMode::Mixnet.to_string(), "mixnet"); + assert_eq!(TestMode::SingleHop.to_string(), "single-hop"); + assert_eq!(TestMode::TwoHop.to_string(), "two-hop"); + assert_eq!(TestMode::LpOnly.to_string(), "lp-only"); + } + + // ============ FromStr tests ============ + + #[test] + fn test_from_str_canonical() { + assert_eq!("mixnet".parse::().unwrap(), TestMode::Mixnet); + assert_eq!( + "single-hop".parse::().unwrap(), + TestMode::SingleHop + ); + assert_eq!("two-hop".parse::().unwrap(), TestMode::TwoHop); + assert_eq!("lp-only".parse::().unwrap(), TestMode::LpOnly); + } + + #[test] + fn test_from_str_alternate_formats() { + // snake_case + assert_eq!( + "single_hop".parse::().unwrap(), + TestMode::SingleHop + ); + assert_eq!("two_hop".parse::().unwrap(), TestMode::TwoHop); + assert_eq!("lp_only".parse::().unwrap(), TestMode::LpOnly); + + // no separator + assert_eq!( + "singlehop".parse::().unwrap(), + TestMode::SingleHop + ); + assert_eq!("twohop".parse::().unwrap(), TestMode::TwoHop); + assert_eq!("lponly".parse::().unwrap(), TestMode::LpOnly); + } + + #[test] + fn test_from_str_case_insensitive() { + assert_eq!("MIXNET".parse::().unwrap(), TestMode::Mixnet); + assert_eq!( + "Single-Hop".parse::().unwrap(), + TestMode::SingleHop + ); + assert_eq!("TWO_HOP".parse::().unwrap(), TestMode::TwoHop); + assert_eq!("LpOnly".parse::().unwrap(), TestMode::LpOnly); + } + + #[test] + fn test_from_str_invalid() { + assert!("invalid".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("mix".parse::().is_err()); + } + + // ============ Roundtrip test ============ + + #[test] + fn test_display_fromstr_roundtrip() { + for mode in [ + TestMode::Mixnet, + TestMode::SingleHop, + TestMode::TwoHop, + TestMode::LpOnly, + ] { + let s = mode.to_string(); + let parsed: TestMode = s.parse().unwrap(); + assert_eq!(mode, parsed); + } + } +} diff --git a/nym-gateway-probe/src/netstack.rs b/nym-gateway-probe/src/netstack.rs index c6d42c04fe0..3c7bd0772ff 100644 --- a/nym-gateway-probe/src/netstack.rs +++ b/nym-gateway-probe/src/netstack.rs @@ -10,6 +10,7 @@ mod sys { unsafe extern "C" { pub unsafe fn wgPing(req: *const c_char) -> *const c_char; + pub unsafe fn wgPingTwoHop(req: *const c_char) -> *const c_char; pub unsafe fn wgFreePtr(ptr: *mut c_void); } } @@ -212,3 +213,71 @@ pub fn ping(req: &NetstackRequestGo) -> anyhow::Result { result } + +/// Request structure for two-hop WireGuard ping test. +/// Matches TwoHopNetstackRequest in Go. +// This struct is serialized to JSON and passed to wgPingTwoHop() via CGO. +// The Go side creates: entry tunnel -> UDP forwarder -> exit tunnel, then runs tests. +#[derive(Clone, Debug, serde::Serialize)] +pub struct TwoHopNetstackRequestGo { + // Entry tunnel configuration (connects directly to entry gateway) + pub entry_wg_ip: String, + pub entry_private_key: String, + pub entry_public_key: String, + pub entry_endpoint: String, + pub entry_awg_args: String, + + // Exit tunnel configuration (connects via forwarder through entry) + pub exit_wg_ip: String, + pub exit_private_key: String, + pub exit_public_key: String, + pub exit_endpoint: String, + pub exit_awg_args: String, + + // Test parameters + pub dns: String, + pub ip_version: u8, + pub ping_hosts: Vec, + pub ping_ips: Vec, + pub num_ping: u8, + pub send_timeout_sec: u64, + pub recv_timeout_sec: u64, + pub download_timeout_sec: u64, +} + +/// Perform a two-hop WireGuard ping test through entry and exit gateways. +/// +/// This creates two nested WireGuard tunnels with a UDP forwarder: +/// - Entry tunnel connects directly to entry gateway (reachable from host) +/// - UDP forwarder listens on localhost and forwards via entry tunnel +/// - Exit tunnel connects to forwarder, traffic flows: exit -> forwarder -> entry -> exit gateway +/// - Tests run through the exit tunnel +pub fn ping_two_hop(req: &TwoHopNetstackRequestGo) -> anyhow::Result { + let req_json = serde_json::to_string_pretty(req)?; + let req_json_cstr = CString::new(req_json)?; + + // SAFETY: safety guarantees are upheld by CGO + let response_str_ptr = unsafe { sys::wgPingTwoHop(req_json_cstr.as_ptr()) }; + if response_str_ptr.is_null() { + return Err(anyhow::anyhow!("wgPingTwoHop() returned null")); + } + + // SAFETY: safety guarantees are upheld by CGO + let response_cstr = unsafe { CStr::from_ptr(response_str_ptr) }; + let result = match response_cstr.to_str() { + Ok(response_str) => { + let mut de = serde_json::Deserializer::from_str(response_str); + let response = NetstackResult::deserialize(&mut de); + + response.context("Failed to deserialize ffi response") + } + Err(err) => Err(anyhow::anyhow!( + "Failed to convert ffi response to utf8 string: {err}" + )), + }; + + // SAFETY: freeing the pointer returned by CGO + unsafe { sys::wgFreePtr(response_str_ptr as _) }; + + result +} diff --git a/nym-gateway-probe/src/run.rs b/nym-gateway-probe/src/run.rs index 05cffed838f..df94d6ef032 100644 --- a/nym-gateway-probe/src/run.rs +++ b/nym-gateway-probe/src/run.rs @@ -1,12 +1,16 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only +use anyhow::bail; use clap::{Parser, Subcommand}; use nym_bin_common::bin_info; use nym_config::defaults::setup_env; use nym_gateway_probe::nodes::{NymApiDirectory, query_gateway_by_ip}; -use nym_gateway_probe::{CredentialArgs, NetstackArgs, ProbeResult, TestedNode}; +use nym_gateway_probe::{ + CredentialArgs, NetstackArgs, ProbeResult, TestMode, TestedNode, TestedNodeDetails, +}; use nym_sdk::mixnet::NodeIdentity; +use std::net::SocketAddr; use std::path::Path; use std::{path::PathBuf, sync::OnceLock}; use tracing::*; @@ -48,6 +52,31 @@ struct CliArgs { #[arg(long, global = true)] exit_gateway_ip: Option, + /// Ed25519 identity of the entry gateway (base58 encoded) + /// When provided, skips HTTP API query - use for localnet testing + #[arg(long, global = true)] + entry_gateway_identity: Option, + + /// Ed25519 identity of the exit gateway (base58 encoded) + /// When provided, skips HTTP API query - use for localnet testing + #[arg(long, global = true)] + exit_gateway_identity: Option, + + /// LP listener address for entry gateway (e.g., "192.168.66.6:41264") + /// Used with --entry-gateway-identity for localnet mode + #[arg(long, global = true)] + entry_lp_address: Option, + + /// LP listener address for exit gateway (e.g., "172.18.0.5:41264") + /// This is the address the entry gateway uses to reach exit (for forwarding) + /// Used with --exit-gateway-identity for localnet mode + #[arg(long, global = true)] + exit_lp_address: Option, + + /// Default LP control port when deriving LP address from gateway IP + #[arg(long, global = true, default_value = "41264")] + lp_port: u16, + /// Identity of the node to test #[arg(long, short, value_parser = validate_node_identity, global = true)] node: Option, @@ -68,6 +97,22 @@ struct CliArgs { #[arg(long, global = true)] test_lp_wg: bool, + /// Test mode - explicitly specify which tests to run + /// + /// Modes: + /// mixnet - Traditional mixnet testing (entry/exit pings + WireGuard via authenticator) + /// single-hop - LP registration + WireGuard on single gateway (no mixnet) + /// two-hop - Entry LP + Exit LP (nested forwarding) + WireGuard tunnel + /// lp-only - LP registration only (no WireGuard) + /// + /// If not specified, mode is inferred from other flags: + /// --only-lp-registration → lp-only + /// --test-lp-wg with exit gateway → two-hop + /// --test-lp-wg without exit → single-hop + /// otherwise → mixnet + #[arg(long, global = true, value_name = "MODE")] + mode: Option, + /// Disable logging during probe #[arg(long, global = true)] ignore_egress_epoch_role: bool, @@ -123,6 +168,40 @@ fn setup_logging() { .init(); } +/// Resolve the test mode from explicit --mode arg or infer from legacy flags +fn resolve_test_mode( + mode_arg: Option<&str>, + only_wireguard: bool, + only_lp_registration: bool, + test_lp_wg: bool, + has_exit_gateway: bool, +) -> anyhow::Result { + if let Some(mode_str) = mode_arg { + // Explicit --mode takes priority + mode_str + .parse::() + .map_err(|e| anyhow::anyhow!("{}", e)) + } else { + // Infer from legacy flags + Ok(TestMode::from_flags( + only_wireguard, + only_lp_registration, + test_lp_wg, + has_exit_gateway, + )) + } +} + +/// Convert TestMode back to legacy boolean flags for backward compatibility +fn mode_to_flags(mode: TestMode) -> (bool, bool, bool) { + match mode { + TestMode::Mixnet => (false, false, false), // only_wireguard handled separately + TestMode::SingleHop => (false, false, true), + TestMode::TwoHop => (false, false, true), + TestMode::LpOnly => (false, true, false), + } +} + pub(crate) async fn run() -> anyhow::Result { let args = CliArgs::parse(); if !args.no_log { @@ -138,48 +217,181 @@ pub(crate) async fn run() -> anyhow::Result { .first() .map(|ep| ep.nyxd_url()) .ok_or(anyhow::anyhow!("missing nyxd url"))?; - // If gateway IP is provided, query it directly without using the directory - let (entry, directory, gateway_node, exit_gateway_node) = if let Some(gateway_ip) = args.gateway_ip { - info!("Using direct IP query mode for gateway: {}", gateway_ip); - let gateway_node = query_gateway_by_ip(gateway_ip).await?; - let identity = gateway_node.identity(); - - // Query exit gateway if provided (for LP forwarding tests) - let exit_node = if let Some(exit_gateway_ip) = args.exit_gateway_ip { - info!("Using direct IP query mode for exit gateway: {}", exit_gateway_ip); - Some(query_gateway_by_ip(exit_gateway_ip).await?) + + // Three resolution modes in priority order: + // 1. Localnet mode: --entry-gateway-identity provided (no HTTP query) + // 2. Direct IP mode: --gateway-ip provided (queries HTTP API) + // 3. Directory mode: uses nym-api directory service + + // Localnet mode: identity provided via CLI, skip HTTP queries entirely + if let Some(entry_identity_str) = &args.entry_gateway_identity { + info!("Using localnet mode with CLI-provided gateway identity"); + + let entry_identity = NodeIdentity::from_base58_string(entry_identity_str)?; + + // Entry LP address: explicit or derived from gateway_ip + lp_port + let entry_lp_addr: SocketAddr = if let Some(lp_addr) = &args.entry_lp_address { + lp_addr + .parse() + .map_err(|e| anyhow::anyhow!("Invalid entry-lp-address '{}': {}", lp_addr, e))? + } else if let Some(gw_ip) = &args.gateway_ip { + // Derive LP address from gateway IP + let ip: std::net::IpAddr = gw_ip + .parse() + .map_err(|e| anyhow::anyhow!("Invalid gateway-ip '{}': {}", gw_ip, e))?; + SocketAddr::new(ip, args.lp_port) } else { - None + anyhow::bail!( + "--entry-lp-address or --gateway-ip required with --entry-gateway-identity" + ); }; - // Still create the directory for potential secondary lookups, - // but only if API URL is available - let directory = if let Some(api_url) = network.endpoints.first().and_then(|ep| ep.api_url()) - { - Some(NymApiDirectory::new(api_url).await?) + let entry_details = TestedNodeDetails::from_cli(entry_identity, entry_lp_addr); + + // Parse exit gateway if provided + let exit_details = if let Some(exit_identity_str) = &args.exit_gateway_identity { + let exit_identity = NodeIdentity::from_base58_string(exit_identity_str)?; + let exit_lp_addr: SocketAddr = args + .exit_lp_address + .as_ref() + .ok_or_else(|| { + anyhow::anyhow!("--exit-lp-address required with --exit-gateway-identity") + })? + .parse() + .map_err(|e| anyhow::anyhow!("Invalid exit-lp-address: {}", e))?; + Some(TestedNodeDetails::from_cli(exit_identity, exit_lp_addr)) } else { None }; - (identity, directory, Some(gateway_node), exit_node) - } else { - // Original behavior: use directory service - let api_url = network - .endpoints - .first() - .and_then(|ep| ep.api_url()) - .ok_or(anyhow::anyhow!("missing api url"))?; + // Resolve test mode from --mode arg or infer from flags + let has_exit = exit_details.is_some(); + let test_mode = resolve_test_mode( + args.mode.as_deref(), + args.only_wireguard, + args.only_lp_registration, + args.test_lp_wg, + has_exit, + )?; + + // Validate that two-hop mode has required exit gateway + if test_mode.needs_exit_gateway() && !has_exit { + bail!( + "--mode two-hop requires exit gateway \ + (use --exit-gateway-identity and --exit-lp-address)" + ); + } - let directory = NymApiDirectory::new(api_url).await?; + info!("Test mode: {}", test_mode); - let entry = if let Some(gateway) = &args.entry_gateway { - NodeIdentity::from_base58_string(gateway)? - } else { - directory.random_exit_with_ipr()? + // Convert back to flags for backward compatibility with existing probe methods + // only_wireguard is preserved from args since it's orthogonal to mode + // (it means "skip ping tests" in mixnet mode, irrelevant for LP modes) + let (_, only_lp_registration, test_lp_wg) = mode_to_flags(test_mode); + let only_wireguard = args.only_wireguard; + + let mut trial = nym_gateway_probe::Probe::new_localnet( + entry_details, + exit_details, + args.netstack_args, + args.credential_args, + ); + + if let Some(awg_args) = args.amnezia_args { + trial.with_amnezia(&awg_args); + } + + // Localnet mode doesn't need directory, but nyxd_url is still used for credentials + return match &args.command { + Some(Commands::RunLocal { + mnemonic, + config_dir, + use_mock_ecash, + }) => { + let config_dir = config_dir + .clone() + .unwrap_or_else(|| Path::new(DEFAULT_CONFIG_DIR).join(&network.network_name)); + + info!( + "using the following directory for the probe config: {}", + config_dir.display() + ); + + Box::pin(trial.probe_run_locally( + &config_dir, + mnemonic.as_deref(), + None, // No directory in localnet mode + nyxd_url, + args.ignore_egress_epoch_role, + only_wireguard, + only_lp_registration, + test_lp_wg, + args.min_gateway_mixnet_performance, + *use_mock_ecash, + )) + .await + } + None => { + Box::pin(trial.probe( + None, // No directory in localnet mode + nyxd_url, + args.ignore_egress_epoch_role, + only_wireguard, + only_lp_registration, + test_lp_wg, + args.min_gateway_mixnet_performance, + )) + .await + } }; + } - (entry, Some(directory), None, None) - }; + // If gateway IP is provided, query it directly without using the directory + let (entry, directory, gateway_node, exit_gateway_node) = + if let Some(gateway_ip) = args.gateway_ip.clone() { + info!("Using direct IP query mode for gateway: {}", gateway_ip); + let gateway_node = query_gateway_by_ip(gateway_ip).await?; + let identity = gateway_node.identity(); + + // Query exit gateway if provided (for LP forwarding tests) + let exit_node = if let Some(exit_gateway_ip) = args.exit_gateway_ip { + info!( + "Using direct IP query mode for exit gateway: {}", + exit_gateway_ip + ); + Some(query_gateway_by_ip(exit_gateway_ip).await?) + } else { + None + }; + + // Still create the directory for potential secondary lookups, + // but only if API URL is available + let directory = + if let Some(api_url) = network.endpoints.first().and_then(|ep| ep.api_url()) { + Some(NymApiDirectory::new(api_url).await?) + } else { + None + }; + + (identity, directory, Some(gateway_node), exit_node) + } else { + // Original behavior: use directory service + let api_url = network + .endpoints + .first() + .and_then(|ep| ep.api_url()) + .ok_or(anyhow::anyhow!("missing api url"))?; + + let directory = NymApiDirectory::new(api_url).await?; + + let entry = if let Some(gateway) = &args.entry_gateway { + NodeIdentity::from_base58_string(gateway)? + } else { + directory.random_exit_with_ipr()? + }; + + (entry, Some(directory), None, None) + }; let test_point = if let Some(node) = args.node { TestedNode::Custom { identity: node } @@ -187,7 +399,24 @@ pub(crate) async fn run() -> anyhow::Result { TestedNode::SameAsEntry }; - let mut trial = if let (Some(entry_node), Some(exit_node)) = (&gateway_node, &exit_gateway_node) { + // Resolve test mode from --mode arg or infer from flags + let has_exit = exit_gateway_node.is_some(); + let test_mode = resolve_test_mode( + args.mode.as_deref(), + args.only_wireguard, + args.only_lp_registration, + args.test_lp_wg, + has_exit, + )?; + info!("Test mode: {}", test_mode); + + // Convert back to flags for backward compatibility with existing probe methods + // only_wireguard is preserved from args since it's orthogonal to mode + let (_, only_lp_registration, test_lp_wg) = mode_to_flags(test_mode); + let only_wireguard = args.only_wireguard; + + let mut trial = if let (Some(entry_node), Some(exit_node)) = (&gateway_node, &exit_gateway_node) + { // Both entry and exit gateways provided (for LP telescoping tests) info!("Using both entry and exit gateways for LP forwarding test"); nym_gateway_probe::Probe::new_with_gateways( @@ -237,9 +466,9 @@ pub(crate) async fn run() -> anyhow::Result { directory, nyxd_url, args.ignore_egress_epoch_role, - args.only_wireguard, - args.only_lp_registration, - args.test_lp_wg, + only_wireguard, + only_lp_registration, + test_lp_wg, args.min_gateway_mixnet_performance, *use_mock_ecash, )) @@ -250,9 +479,9 @@ pub(crate) async fn run() -> anyhow::Result { directory, nyxd_url, args.ignore_egress_epoch_role, - args.only_wireguard, - args.only_lp_registration, - args.test_lp_wg, + only_wireguard, + only_lp_registration, + test_lp_wg, args.min_gateway_mixnet_performance, )) .await diff --git a/nym-registration-client/src/lib.rs b/nym-registration-client/src/lib.rs index 4b023ea2f19..f8eb603ea2c 100644 --- a/nym-registration-client/src/lib.rs +++ b/nym-registration-client/src/lib.rs @@ -189,14 +189,13 @@ impl RegistrationClient { ); // Perform handshake with entry gateway (outer session now established) - entry_client - .perform_handshake() - .await - .map_err(|source| RegistrationClientError::EntryGatewayRegisterLp { + entry_client.perform_handshake().await.map_err(|source| { + RegistrationClientError::EntryGatewayRegisterLp { gateway_id: self.config.entry.node.identity.to_base58_string(), lp_address: entry_lp_address, source: Box::new(source), - })?; + } + })?; tracing::info!("Outer session with entry gateway established"); diff --git a/nym-registration-client/src/lp_client/client.rs b/nym-registration-client/src/lp_client/client.rs index 210b7323da4..251d83df00a 100644 --- a/nym-registration-client/src/lp_client/client.rs +++ b/nym-registration-client/src/lp_client/client.rs @@ -10,7 +10,7 @@ use nym_bandwidth_controller::{BandwidthTicketProvider, DEFAULT_TICKETS_TO_SPEND use nym_credentials_interface::{CredentialSpendingData, TicketType}; use nym_crypto::asymmetric::{ed25519, x25519}; use nym_lp::LpPacket; -use nym_lp::codec::{parse_lp_packet, serialize_lp_packet, OuterAeadKey}; +use nym_lp::codec::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; use nym_lp::message::ForwardPacketData; use nym_lp::state_machine::{LpAction, LpInput, LpStateMachine}; use nym_registration_common::{GatewayData, LpRegistrationRequest, LpRegistrationResponse}; @@ -207,7 +207,8 @@ impl LpRegistrationClient { let ack_response = Self::connect_send_receive( self.gateway_lp_address, &client_hello_packet, - None, // No outer key before handshake + None, // No outer key for ClientHello (before PSK) + None, // No outer key for Ack response (before PSK) &self.config, ) .await?; @@ -258,25 +259,35 @@ impl LpRegistrationClient { loop { // Send pending packet if we have one if let Some(packet) = pending_packet.take() { - // Get outer key from session (None before PSK, Some after) - let outer_key = state_machine + // Get outer keys from session: + // - send_key: outer_aead_key_for_sending() returns None until PSQ complete + // - recv_key: outer_aead_key() returns key as soon as PSK is derived + let send_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let recv_key = state_machine .session() .ok() .and_then(|s| s.outer_aead_key()); - tracing::trace!("Sending handshake packet (outer_key={})", outer_key.is_some()); + tracing::trace!( + "Sending handshake packet (send_key={}, recv_key={})", + send_key.is_some(), + recv_key.is_some() + ); let response = Self::connect_send_receive( self.gateway_lp_address, &packet, - outer_key.as_ref(), + send_key.as_ref(), + recv_key.as_ref(), &self.config, ) .await?; tracing::trace!("Received handshake response"); // Process the received packet - if let Some(action) = - state_machine.process_input(LpInput::ReceivePacket(response)) + if let Some(action) = state_machine.process_input(LpInput::ReceivePacket(response)) { match action? { LpAction::SendPacket(response_packet) => { @@ -287,7 +298,11 @@ impl LpRegistrationClient { if state_machine.session()?.is_handshake_complete() { // Send the final packet before breaking if let Some(final_packet) = pending_packet.take() { - let outer_key = state_machine + let send_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let recv_key = state_machine .session() .ok() .and_then(|s| s.outer_aead_key()); @@ -295,7 +310,8 @@ impl LpRegistrationClient { let ack_response = Self::connect_send_receive( self.gateway_lp_address, &final_packet, - outer_key.as_ref(), + send_key.as_ref(), + recv_key.as_ref(), &self.config, ) .await?; @@ -330,10 +346,10 @@ impl LpRegistrationClient { .session()? .prepare_handshake_message() .ok_or_else(|| { - LpClientError::Transport( - "No handshake message available after KKT".to_string(), - ) - })??; + LpClientError::Transport( + "No handshake message available after KKT".to_string(), + ) + })??; let noise_packet = state_machine.session()?.next_packet(noise_msg)?; pending_packet = Some(noise_packet); } @@ -369,10 +385,22 @@ impl LpRegistrationClient { /// /// # Errors /// Returns an error if connection, send, or receive fails. + /// + /// # Outer AEAD Keys + /// + /// Send and receive use separate keys because during the PSQ handshake: + /// - Initiator derives PSK when preparing msg 1, but must send it cleartext + /// (responder hasn't derived PSK yet) + /// - Responder sends msg 2 encrypted (both have PSK now) + /// - Initiator can decrypt msg 2 (has had PSK since preparing msg 1) + /// + /// Use `outer_aead_key_for_sending()` for `send_key` (gates on PSQ completion) + /// and `outer_aead_key()` for `recv_key` (available as soon as PSK derived). async fn connect_send_receive( address: SocketAddr, packet: &LpPacket, - outer_key: Option<&OuterAeadKey>, + send_key: Option<&OuterAeadKey>, + recv_key: Option<&OuterAeadKey>, config: &LpConfig, ) -> Result { // 1. Connect with timeout @@ -398,11 +426,11 @@ impl LpRegistrationClient { source, })?; - // 3. Send packet with optional outer AEAD - Self::send_packet_with_key(&mut stream, packet, outer_key).await?; + // 3. Send packet with send_key + Self::send_packet_with_key(&mut stream, packet, send_key).await?; - // 4. Receive response with optional outer AEAD - let response = Self::receive_packet_with_key(&mut stream, outer_key).await?; + // 4. Receive response with recv_key + let response = Self::receive_packet_with_key(&mut stream, recv_key).await?; // Connection drops when stream goes out of scope Ok(response) @@ -570,9 +598,7 @@ impl LpRegistrationClient { ) -> Result { // Ensure handshake is complete (state machine exists) let state_machine = self.state_machine.as_mut().ok_or_else(|| { - LpClientError::Transport( - "Cannot register: handshake not completed".to_string(), - ) + LpClientError::Transport("Cannot register: handshake not completed".to_string()) })?; tracing::debug!("Sending registration request (packet-per-connection)"); @@ -617,8 +643,12 @@ impl LpRegistrationClient { } }; - // 4. Get outer key from session - let outer_key = state_machine + // 4. Get outer keys from session + let send_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let recv_key = state_machine .session() .ok() .and_then(|s| s.outer_aead_key()); @@ -629,7 +659,8 @@ impl LpRegistrationClient { Self::connect_send_receive( self.gateway_lp_address, &request_packet, - outer_key.as_ref(), + send_key.as_ref(), + recv_key.as_ref(), &self.config, ), ) @@ -797,8 +828,12 @@ impl LpRegistrationClient { } }; - // 4. Get outer key from session - let outer_key = state_machine + // 4. Get outer keys from session + let send_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let recv_key = state_machine .session() .ok() .and_then(|s| s.outer_aead_key()); @@ -807,7 +842,8 @@ impl LpRegistrationClient { let response_packet = Self::connect_send_receive( self.gateway_lp_address, &forward_packet, - outer_key.as_ref(), + send_key.as_ref(), + recv_key.as_ref(), &self.config, ) .await?; diff --git a/nym-registration-client/src/lp_client/nested_session.rs b/nym-registration-client/src/lp_client/nested_session.rs index 5f462bd0ae5..88096683e17 100644 --- a/nym-registration-client/src/lp_client/nested_session.rs +++ b/nym-registration-client/src/lp_client/nested_session.rs @@ -24,7 +24,7 @@ use bytes::BytesMut; use nym_bandwidth_controller::BandwidthTicketProvider; use nym_credentials_interface::TicketType; use nym_crypto::asymmetric::{ed25519, x25519}; -use nym_lp::codec::{parse_lp_packet, serialize_lp_packet, OuterAeadKey}; +use nym_lp::codec::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; use nym_lp::state_machine::{LpAction, LpInput, LpStateMachine}; use nym_lp::{LpMessage, LpPacket}; use nym_registration_common::{GatewayData, LpRegistrationRequest, LpRegistrationResponse}; @@ -110,23 +110,16 @@ impl NestedLpSession { /// - Forwarding through entry gateway fails /// - Exit gateway handshake fails /// - Cryptographic operations fail - async fn perform_handshake( - &mut self, - outer_client: &mut LpRegistrationClient, - ) -> Result<()> { + async fn perform_handshake(&mut self, outer_client: &mut LpRegistrationClient) -> Result<()> { tracing::debug!( "Starting nested LP handshake with exit gateway {}", self.exit_address ); // Step 1: Derive X25519 keys from Ed25519 for Noise protocol - let client_x25519_public = self - .client_keypair - .public_key() - .to_x25519() - .map_err(|e| { - LpClientError::Crypto(format!("Failed to derive X25519 public key: {}", e)) - })?; + let client_x25519_public = self.client_keypair.public_key().to_x25519().map_err(|e| { + LpClientError::Crypto(format!("Failed to derive X25519 public key: {}", e)) + })?; // Step 2: Generate ClientHello for exit gateway let client_hello_data = nym_lp::ClientHelloData::new_with_fresh_salt( @@ -144,7 +137,7 @@ impl NestedLpSession { // Step 3: Send ClientHello to exit gateway via forwarding let client_hello_header = nym_lp::packet::LpHeader::new( nym_lp::BOOTSTRAP_RECEIVER_IDX, // Use constant for bootstrap session - 0, // counter starts at 0 + 0, // counter starts at 0 ); let client_hello_packet = nym_lp::LpPacket::new( client_hello_header, @@ -219,8 +212,7 @@ impl NestedLpSession { tracing::trace!("Received handshake response from exit"); // Process the received packet - if let Some(action) = - state_machine.process_input(LpInput::ReceivePacket(response)) + if let Some(action) = state_machine.process_input(LpInput::ReceivePacket(response)) { match action? { LpAction::SendPacket(response_packet) => { @@ -238,9 +230,7 @@ impl NestedLpSession { ) .await?; } - tracing::info!( - "Nested LP handshake completed with exit gateway" - ); + tracing::info!("Nested LP handshake completed with exit gateway"); break; } } @@ -255,10 +245,10 @@ impl NestedLpSession { .session()? .prepare_handshake_message() .ok_or_else(|| { - LpClientError::Transport( - "No handshake message available after KKT".to_string(), - ) - })??; + LpClientError::Transport( + "No handshake message available after KKT".to_string(), + ) + })??; let noise_packet = state_machine.session()?.next_packet(noise_msg)?; pending_packet = Some(noise_packet); } @@ -280,6 +270,161 @@ impl NestedLpSession { Ok(()) } + /// Performs handshake and registration with the exit gateway via forwarding, + /// using a pre-made credential. + /// + /// This variant is useful for mock ecash testing where the credential is provided + /// directly instead of being acquired from a bandwidth controller. + /// + /// # Arguments + /// * `outer_client` - Connected LP client with established outer session to entry gateway + /// * `wg_keypair` - Client's WireGuard x25519 keypair + /// * `credential` - Pre-made bandwidth credential (e.g., mock ecash) + /// * `ticket_type` - Type of bandwidth ticket to use + /// * `client_ip` - Client IP address for registration metadata + /// + /// # Returns + /// * `Ok(GatewayData)` - Exit gateway configuration data on successful registration + pub async fn handshake_and_register_with_credential( + &mut self, + outer_client: &mut LpRegistrationClient, + wg_keypair: &x25519::KeyPair, + credential: nym_credentials_interface::CredentialSpendingData, + ticket_type: TicketType, + client_ip: IpAddr, + ) -> Result { + // Step 1: Perform handshake with exit gateway via forwarding + self.perform_handshake(outer_client).await?; + + // Step 2: Get the state machine (must exist after successful handshake) + let state_machine = self.state_machine.as_mut().ok_or_else(|| { + LpClientError::Transport("State machine missing after handshake".to_string()) + })?; + + tracing::debug!( + "Building registration request for exit gateway (with pre-made credential)" + ); + + // Step 3: Build registration request (credential already provided) + let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into()); + let request = + LpRegistrationRequest::new_dvpn(wg_public_key, credential, ticket_type, client_ip); + + tracing::trace!("Built registration request: {:?}", request); + + // Step 4: Serialize the request + let request_bytes = bincode::serialize(&request).map_err(|e| { + LpClientError::Transport(format!("Failed to serialize registration request: {}", e)) + })?; + + tracing::debug!( + "Sending registration request to exit gateway via forwarding ({} bytes)", + request_bytes.len() + ); + + // Step 5: Encrypt and prepare packet via state machine + let action = state_machine + .process_input(LpInput::SendData(request_bytes)) + .ok_or_else(|| { + LpClientError::Transport("State machine returned no action".to_string()) + })? + .map_err(|e| { + LpClientError::Transport(format!("Failed to encrypt registration request: {}", e)) + })?; + + // Step 6: Send the encrypted packet via forwarding + // Get outer key for AEAD encryption (PSK is available after handshake) + let outer_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let response_bytes = match action { + LpAction::SendPacket(packet) => { + let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?; + outer_client + .send_forward_packet( + self.exit_identity, + self.exit_address.clone(), + packet_bytes, + ) + .await? + } + other => { + return Err(LpClientError::Transport(format!( + "Unexpected action when sending registration data: {:?}", + other + ))); + } + }; + + tracing::trace!("Received registration response from exit gateway"); + + // Step 7: Parse response bytes to LP packet + let outer_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key()); + let response_packet = Self::parse_packet(&response_bytes, outer_key.as_ref())?; + + // Step 8: Decrypt via state machine + let action = state_machine + .process_input(LpInput::ReceivePacket(response_packet)) + .ok_or_else(|| { + LpClientError::Transport("State machine returned no action".to_string()) + })? + .map_err(|e| { + LpClientError::Transport(format!("Failed to decrypt registration response: {}", e)) + })?; + + // Step 9: Extract decrypted data + let response_data = match action { + LpAction::DeliverData(data) => data, + other => { + return Err(LpClientError::Transport(format!( + "Unexpected action when receiving registration response: {:?}", + other + ))); + } + }; + + // Step 10: Deserialize the response + let response: LpRegistrationResponse = + bincode::deserialize(&response_data).map_err(|e| { + LpClientError::Transport(format!( + "Failed to deserialize registration response: {}", + e + )) + })?; + + tracing::debug!( + "Received registration response from exit: success={}", + response.success, + ); + + // Step 11: Validate and extract GatewayData + if !response.success { + let error_msg = response + .error + .unwrap_or_else(|| "Unknown error".to_string()); + tracing::warn!("Exit gateway rejected registration: {}", error_msg); + return Err(LpClientError::RegistrationRejected { reason: error_msg }); + } + + // Extract gateway_data + let gateway_data = response.gateway_data.ok_or_else(|| { + LpClientError::Transport( + "Gateway response missing gateway_data despite success=true".to_string(), + ) + })?; + + tracing::info!( + "Exit gateway registration successful! Allocated bandwidth: {} bytes", + response.allocated_bandwidth + ); + + Ok(gateway_data) + } + /// Performs handshake and registration with the exit gateway via forwarding. /// /// This is the main entry point for nested LP registration. It: @@ -328,19 +473,21 @@ impl NestedLpSession { // Step 3: Acquire bandwidth credential let credential = bandwidth_controller - .get_ecash_ticket(ticket_type, *gateway_identity, nym_bandwidth_controller::DEFAULT_TICKETS_TO_SPEND) + .get_ecash_ticket( + ticket_type, + *gateway_identity, + nym_bandwidth_controller::DEFAULT_TICKETS_TO_SPEND, + ) .await .map_err(|e| { - LpClientError::Transport(format!( - "Failed to acquire bandwidth credential: {}", - e - )) + LpClientError::Transport(format!("Failed to acquire bandwidth credential: {}", e)) })? .data; // Step 4: Build registration request let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into()); - let request = LpRegistrationRequest::new_dvpn(wg_public_key, credential, ticket_type, client_ip); + let request = + LpRegistrationRequest::new_dvpn(wg_public_key, credential, ticket_type, client_ip); tracing::trace!("Built registration request: {:?}", request); @@ -361,15 +508,15 @@ impl NestedLpSession { LpClientError::Transport("State machine returned no action".to_string()) })? .map_err(|e| { - LpClientError::Transport(format!( - "Failed to encrypt registration request: {}", - e - )) + LpClientError::Transport(format!("Failed to encrypt registration request: {}", e)) })?; // Step 7: Send the encrypted packet via forwarding // Get outer key for AEAD encryption (PSK is available after handshake) - let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key()); + let outer_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); let response_bytes = match action { LpAction::SendPacket(packet) => { let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?; @@ -392,7 +539,10 @@ impl NestedLpSession { tracing::trace!("Received registration response from exit gateway"); // Step 8: Parse response bytes to LP packet - let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key()); + let outer_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key()); let response_packet = Self::parse_packet(&response_bytes, outer_key.as_ref())?; // Step 9: Decrypt via state machine @@ -402,10 +552,7 @@ impl NestedLpSession { LpClientError::Transport("State machine returned no action".to_string()) })? .map_err(|e| { - LpClientError::Transport(format!( - "Failed to decrypt registration response: {}", - e - )) + LpClientError::Transport(format!("Failed to decrypt registration response: {}", e)) })?; // Step 10: Extract decrypted data @@ -470,16 +617,20 @@ impl NestedLpSession { state_machine: &LpStateMachine, packet: &LpPacket, ) -> Result { - let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key()); - let packet_bytes = Self::serialize_packet(packet, outer_key.as_ref())?; + // Use outer_aead_key_for_sending() for send, outer_aead_key() for receive + let send_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key_for_sending()); + let packet_bytes = Self::serialize_packet(packet, send_key.as_ref())?; let response_bytes = outer_client - .send_forward_packet( - self.exit_identity, - self.exit_address.clone(), - packet_bytes, - ) + .send_forward_packet(self.exit_identity, self.exit_address.clone(), packet_bytes) .await?; - Self::parse_packet(&response_bytes, outer_key.as_ref()) + let recv_key = state_machine + .session() + .ok() + .and_then(|s| s.outer_aead_key()); + Self::parse_packet(&response_bytes, recv_key.as_ref()) } /// Serializes an LP packet to bytes. @@ -513,8 +664,7 @@ impl NestedLpSession { /// Returns an error if parsing fails fn parse_packet(bytes: &[u8], outer_key: Option<&OuterAeadKey>) -> Result { // Use outer AEAD key when available (after PSK derivation) - parse_lp_packet(bytes, outer_key).map_err(|e| { - LpClientError::Transport(format!("Failed to parse LP packet: {}", e)) - }) + parse_lp_packet(bytes, outer_key) + .map_err(|e| LpClientError::Transport(format!("Failed to parse LP packet: {}", e))) } } diff --git a/scripts/probe-localnet.sh b/scripts/probe-localnet.sh new file mode 100755 index 00000000000..ab96ab0d88d --- /dev/null +++ b/scripts/probe-localnet.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Probe localnet gateways for LP two-hop testing +# Usage: ./scripts/probe-localnet.sh [mode] +# Modes: two-hop (default), single-hop, lp-only + +set -e + +MODE="${1:-two-hop}" + +# Gateway API (localhost mapped ports) +ENTRY_API="127.0.0.1:30004" +EXIT_API="127.0.0.1:30005" + +# Get gateway identities from API +ENTRY_ID=$(curl -s "http://${ENTRY_API}/api/v1/host-information" | jq -r '.data.keys.ed25519_identity') +EXIT_ID=$(curl -s "http://${EXIT_API}/api/v1/host-information" | jq -r '.data.keys.ed25519_identity') + +if [ -z "$ENTRY_ID" ] || [ "$ENTRY_ID" = "null" ] || [ -z "$EXIT_ID" ] || [ "$EXIT_ID" = "null" ]; then + echo "Error: Could not get gateway identities from API" + echo "Make sure localnet is running: container list" + exit 1 +fi + +echo "Entry gateway: $ENTRY_ID" +echo "Exit gateway: $EXIT_ID" +echo "Mode: $MODE" +echo "---" + +cargo run -p nym-gateway-probe -- run-local \ + --entry-gateway-identity "$ENTRY_ID" \ + --entry-lp-address '127.0.0.1:41264' \ + --exit-gateway-identity "$EXIT_ID" \ + --exit-lp-address '192.168.65.6:41264' \ + --mode "$MODE" \ + --use-mock-ecash