fix: handle fragmented ClientHello in auto-sense server

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Unreleased
 
+  * Fix server auto-sensing DTLS version with fragmented ClientHello #87
   * DTLS 1.2 DTLS 1.3, parser reject ApplicationData in epoch 0/plaintext #90
   * DTLS 1.3 reject plaintext records with non-zero epoch #90
   * Silently discard invalid records and process subsequent valid records #90

src/detect.rs → src/auto.rs

@@ -1,14 +1,20 @@
 /// Version detection and hybrid ClientHello for auto-sensing DTLS endpoints.
 ///
-/// Both `client_hello_version` and `server_hello_version` do lightweight
-/// parsing of the first record in a datagram, looking for the
-/// `supported_versions` extension to decide between DTLS 1.2 and 1.3.
+/// `server_hello_version` does lightweight parsing of the first record in a
+/// datagram. It looks for a `HelloVerifyRequest` or a `ServerHello` with the
+/// `supported_versions` extension to decide between DTLS 1.2 and 1.3 for the
+/// client auto-sense path.
 ///
 /// [`HybridClientHello`] constructs a ClientHello compatible with both
-/// DTLS 1.2 and 1.3 servers: it offers both version's cipher suites and
+/// DTLS 1.2 and 1.3 servers: it offers both versions cipher suites and
 /// includes `supported_versions` with both versions. Once the server
 /// responds, the caller inspects the reply with `server_hello_version`
 /// and forks into the appropriate handshake path.
+///
+/// Server-side auto-sense does not use lightweight detection. Instead,
+/// it starts a DTLS 1.3 server which handles fragment reassembly natively
+/// and falls back to DTLS 1.2 via [`Error::Dtls12Fallback`] if the
+/// reassembled ClientHello does not offer DTLS 1.3.
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -245,35 +251,6 @@ impl HybridClientHello {
     }
 }
 
-/// Auto-sense server waiting for the first ClientHello to determine the DTLS version.
-pub(crate) struct ServerPending {
-    config: Arc<Config>,
-    certificate: DtlsCertificate,
-    last_now: Instant,
-}
-
-impl ServerPending {
-    pub fn new(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Self {
-        ServerPending {
-            config,
-            certificate,
-            last_now: now,
-        }
-    }
-
-    pub fn handle_timeout(&mut self, now: Instant) {
-        self.last_now = now;
-    }
-
-    pub fn poll_output<'a>(&self, _buf: &'a mut [u8]) -> Output<'a> {
-        Output::Timeout(self.last_now + Duration::from_secs(86400))
-    }
-
-    pub fn into_parts(self) -> (Arc<Config>, DtlsCertificate, Instant) {
-        (self.config, self.certificate, self.last_now)
-    }
-}
-
 /// Auto-sense client that sends a hybrid ClientHello and waits for the server's response
 /// to determine the DTLS version.
 pub(crate) struct ClientPending {
@@ -373,117 +350,6 @@ pub(crate) enum DetectedVersion {
     Unknown,
 }
 
-/// Detect DTLS version from a ClientHello packet.
-///
-/// Returns `Dtls13` if the `supported_versions` extension contains
-/// DTLS 1.3 (0xFEFC), otherwise `Dtls12`.
-pub(crate) fn client_hello_version(packet: &[u8]) -> DetectedVersion {
-    match client_hello_version_inner(packet) {
-        Some(true) => DetectedVersion::Dtls13,
-        _ => DetectedVersion::Dtls12,
-    }
-}
-
-fn client_hello_version_inner(packet: &[u8]) -> Option<bool> {
-    // Record header: content_type(1) + version(2) + epoch(2) + seq(6) + length(2) = 13
-    if packet.len() < 13 {
-        return Some(false);
-    }
-
-    // content_type must be 0x16 (Handshake)
-    if packet[0] != 0x16 {
-        return Some(false);
-    }
-
-    let record_len = u16::from_be_bytes([packet[11], packet[12]]) as usize;
-    let record_body = packet.get(13..13 + record_len)?;
-
-    // Handshake header: msg_type(1) + length(3) + message_seq(2) +
-    //   fragment_offset(3) + fragment_length(3) = 12
-    if record_body.len() < 12 {
-        return Some(false);
-    }
-
-    // msg_type must be 1 (ClientHello)
-    if record_body[0] != 1 {
-        return Some(false);
-    }
-
-    let fragment_len = ((record_body[9] as usize) << 16)
-        | ((record_body[10] as usize) << 8)
-        | (record_body[11] as usize);
-    let body = record_body.get(12..12 + fragment_len)?;
-
-    // ClientHello body:
-    //   client_version(2) + random(32) = 34 minimum before session_id
-    if body.len() < 34 {
-        return Some(false);
-    }
-    let mut pos = 34;
-
-    // session_id: 1-byte length + data
-    let sid_len = *body.get(pos)? as usize;
-    pos += 1 + sid_len;
-
-    // cookie: 1-byte length + data
-    let cookie_len = *body.get(pos)? as usize;
-    pos += 1 + cookie_len;
-
-    // cipher_suites: 2-byte length + data
-    if pos + 2 > body.len() {
-        return Some(false);
-    }
-    let cs_len = u16::from_be_bytes([body[pos], body[pos + 1]]) as usize;
-    pos += 2 + cs_len;
-
-    // compression_methods: 1-byte length + data
-    let cm_len = *body.get(pos)? as usize;
-    pos += 1 + cm_len;
-
-    // extensions: 2-byte total length
-    if pos + 2 > body.len() {
-        return Some(false);
-    }
-    let ext_total_len = u16::from_be_bytes([body[pos], body[pos + 1]]) as usize;
-    pos += 2;
-    let ext_end = pos + ext_total_len;
-    if ext_end > body.len() {
-        return Some(false);
-    }
-
-    // Walk extensions looking for supported_versions (0x002B)
-    while pos + 4 <= ext_end {
-        let ext_type = u16::from_be_bytes([body[pos], body[pos + 1]]);
-        let ext_len = u16::from_be_bytes([body[pos + 2], body[pos + 3]]) as usize;
-        pos += 4;
-
-        if ext_type == 0x002B {
-            // supported_versions client format: 1-byte list length, then 2-byte versions
-            if ext_len < 1 {
-                return Some(false);
-            }
-            let list_len = body[pos] as usize;
-            let list_start = pos + 1;
-            if list_start + list_len > pos + ext_len {
-                return Some(false);
-            }
-            let mut i = list_start;
-            while i + 2 <= list_start + list_len {
-                let version = u16::from_be_bytes([body[i], body[i + 1]]);
-                if version == 0xFEFC {
-                    return Some(true);
-                }
-                i += 2;
-            }
-            return Some(false);
-        }
-
-        pos += ext_len;
-    }
-
-    Some(false)
-}
-
 /// Detect DTLS version from a server response (ServerHello or HelloVerifyRequest).
 ///
 /// - HelloVerifyRequest (msg_type 3) → `Dtls12`

src/dtls13/client.rs

@@ -170,7 +170,7 @@ impl Client {
     /// already sent on the wire by `ClientPending`, so no record is
     /// enqueued for output.
     pub(crate) fn new_from_hybrid(
-        hybrid: crate::detect::HybridClientHello,
+        hybrid: crate::auto::HybridClientHello,
         config: std::sync::Arc<crate::Config>,
         certificate: crate::DtlsCertificate,
         now: Instant,
@@ -210,7 +210,7 @@ impl Client {
     }
 
     pub fn into_server(self) -> Server {
-        Server::new_with_engine(self.engine, self.last_now)
+        Server::new_with_engine(self.engine, self.last_now, false)
     }
 
     pub(crate) fn state_name(&self) -> &'static str {

src/dtls13/engine.rs

@@ -27,7 +27,7 @@ use crate::dtls13::message::Sequence;
 use crate::timer::ExponentialBackoff;
 use crate::types::{HashAlgorithm, Random};
 use crate::window::ReplayWindow;
-use crate::{Config, Error, Output, SeededRng};
+use crate::{Config, DtlsCertificate, Error, Output, SeededRng};
 
 const MAX_DEFRAGMENT_PACKETS: usize = 50;
 
@@ -39,6 +39,9 @@ pub struct Engine {
     /// Configuration options.
     config: Arc<Config>,
 
+    /// Saved certificate
+    certificate: DtlsCertificate,
+
     /// Seedable random number generator for deterministic testing
     rng: SeededRng,
 
@@ -93,9 +96,6 @@ pub struct Engine {
     /// Whether the remote peer has enabled encryption
     peer_encryption_enabled: bool,
 
-    /// Certificate in DER format
-    certificate_der: Vec<u8>,
-
     /// Signing key for CertificateVerify
     signing_key: Box<dyn SigningKey>,
 
@@ -189,7 +189,7 @@ struct Entry {
 }
 
 impl Engine {
-    pub fn new(config: Arc<Config>, certificate: crate::DtlsCertificate) -> Self {
+    pub fn new(config: Arc<Config>, certificate: DtlsCertificate) -> Self {
         let mut rng = SeededRng::new(config.rng_seed());
 
         let flight_backoff =
@@ -206,6 +206,7 @@ impl Engine {
 
         Self {
             config,
+            certificate,
             rng,
             buffers_free: BufferPool::default(),
             sequence_epoch_0: Sequence::new(0),
@@ -224,7 +225,6 @@ impl Engine {
             prev_app_send_seq: 0,
             app_recv_keys: ArrayVec::new(),
             peer_encryption_enabled: false,
-            certificate_der: certificate.certificate,
             signing_key,
             is_client: false,
             peer_handshake_seq_no: 0,
@@ -246,6 +246,10 @@ impl Engine {
         }
     }
 
+    pub fn into_fallback(self) -> (Arc<Config>, DtlsCertificate) {
+        (self.config, self.certificate)
+    }
+
     pub fn set_client(&mut self, is_client: bool) {
         self.is_client = is_client;
     }
@@ -307,7 +311,7 @@ impl Engine {
     }
 
     pub fn certificate_der(&self) -> &[u8] {
-        &self.certificate_der
+        &self.certificate.certificate
     }
 
     pub fn signing_key(&mut self) -> &mut dyn SigningKey {

src/dtls13/server.rs

@@ -74,6 +74,8 @@ const HRR_RANDOM: [u8; 32] = [
     0xC2, 0xA2, 0x11, 0x16, 0x7A, 0xBB, 0x8C, 0x5E, 0x07, 0x9E, 0x09, 0xE2, 0xC8, 0xA8, 0x33, 0x9C,
 ];
 
+const MAX_RETAINED_CLIENT_HELLO: usize = 64;
+
 /// DTLS 1.3 server
 pub struct Server {
     /// Current server state.
@@ -135,6 +137,15 @@ pub struct Server {
 
     /// Whether we need to respond with our own KeyUpdate.
     pending_key_update_response: bool,
+
+    /// When true, a ClientHello without DTLS 1.3 in `supported_versions`
+    /// returns [`Error::Dtls12Fallback`] instead of a security error.
+    /// Used by the auto-sense server path.
+    auto_mode: bool,
+
+    /// Raw packets buffered during auto-sense so they can be replayed
+    /// to a DTLS 1.2 server on fallback.
+    retained_hello: VecDeque<Buf>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -156,10 +167,21 @@ impl Server {
     /// Create a new DTLS 1.3 server.
     pub fn new(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Server {
         let engine = Engine::new(config, certificate);
-        Self::new_with_engine(engine, now)
+        Self::new_with_engine(engine, now, false)
     }
 
-    pub(crate) fn new_with_engine(mut engine: Engine, now: Instant) -> Server {
+    /// Create a new DTLS 1.3 server in auto-sense mode.
+    ///
+    /// In auto-sense mode, if the ClientHello does not offer DTLS 1.3
+    /// in `supported_versions`, the server returns [`Error::Dtls12Fallback`]
+    /// instead of a fatal security error, allowing the caller to switch
+    /// to a DTLS 1.2 server.
+    pub fn new_auto(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Server {
+        let engine = Engine::new(config, certificate);
+        Self::new_with_engine(engine, now, true)
+    }
+
+    pub fn new_with_engine(mut engine: Engine, now: Instant, auto_mode: bool) -> Server {
         let cookie_secret = engine.random_arr();
 
         Server {
@@ -183,20 +205,55 @@ impl Server {
             hello_retry: false,
             cookie_secret,
             pending_key_update_response: false,
+            auto_mode,
+            retained_hello: VecDeque::with_capacity(10),
         }
     }
 
     pub fn into_client(self) -> Client {
         Client::new_with_engine(self.engine, self.last_now)
     }
 
+    /// Whether this server is in auto-sense mode.
+    pub fn is_auto_mode(&self) -> bool {
+        self.auto_mode
+    }
+
+    /// Take all relevant config from this server instance.
+    ///
+    /// This is used in two cases:
+    ///
+    /// 1. Switching a server pending (auto-mode) to dtls12 server
+    /// 2. set_active(true), turning a server pending (auto-mode) to a ClientPending
+    pub fn into_parts(self) -> (Arc<Config>, DtlsCertificate, Instant, VecDeque<Buf>) {
+        let (config, cert) = self.engine.into_fallback();
+        (config, cert, self.last_now, self.retained_hello)
+    }
+
     pub(crate) fn state_name(&self) -> &'static str {
         self.state.name()
     }
 
     pub fn handle_packet(&mut self, packet: &[u8]) -> Result<(), Error> {
+        // In auto-sense mode, buffer raw packets while still waiting for
+        // the ClientHello so they can be replayed to Server12 on fallback.
+        if self.auto_mode && self.state == State::AwaitClientHello {
+            // Cap buffered fragments to prevent unbounded growth from malicious traffic
+            if self.retained_hello.len() >= MAX_RETAINED_CLIENT_HELLO {
+                return Err(Error::TooManyClientHelloFragments);
+            }
+            self.retained_hello.push_back(packet.to_buf());
+        }
+
         self.engine.parse_packet(packet)?;
         self.make_progress()?;
+
+        // Once past AwaitClientHello, DTLS 1.3 is committed — free the buffer.
+        if self.auto_mode && self.state != State::AwaitClientHello {
+            self.retained_hello.clear();
+            self.auto_mode = false;
+        }
+
         Ok(())
     }
 
@@ -392,6 +449,9 @@ impl State {
         }
 
         if !supported_versions_ok {
+            if server.auto_mode {
+                return Err(Error::Dtls12Fallback);
+            }
             return Err(Error::SecurityError(
                 "ClientHello must include DTLS 1.3 in supported_versions".to_string(),
             ));