diff --git a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs index f228f68d961..b1981c79e1a 100644 --- a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs +++ b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs @@ -273,6 +273,7 @@ mod api_impl { use sled_agent_types_versions::v26; use sled_agent_types_versions::v30; use sled_agent_types_versions::v33; + use sled_agent_types_versions::v38; use sled_diagnostics::SledDiagnosticsQueryOutput; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -773,6 +774,13 @@ mod api_impl { unimplemented!() } + async fn write_network_bootstore_config_v38( + _rqctx: RequestContext, + _body: TypedBody, + ) -> Result { + unimplemented!() + } + async fn write_network_bootstore_config_v33( _rqctx: RequestContext, _body: TypedBody, diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index ba76de44053..f56720000a3 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -52,7 +52,6 @@ use omicron_common::{ use rdb_types::{Prefix, Prefix4, Prefix6}; use serde_json::json; use sled_agent_client::types::HostPortConfig; -use sled_agent_types::early_networking::BfdPeerConfig; use sled_agent_types::early_networking::BgpConfig as SledBgpConfig; use sled_agent_types::early_networking::BgpPeerConfig as SledBgpPeerConfig; use sled_agent_types::early_networking::EarlyNetworkConfigEnvelope; @@ -71,6 +70,10 @@ use sled_agent_types::early_networking::UplinkAddress; use sled_agent_types::early_networking::UplinkAddressConfig; use sled_agent_types::system_networking::SystemNetworkingConfig; use sled_agent_types::system_networking::WriteNetworkConfigRequest; +use sled_agent_types::{ + early_networking::BfdPeerConfig, + system_networking::BlueprintExternalNetworkingConfig, +}; use slog_error_chain::InlineErrorChain; use std::{ collections::{HashMap, HashSet, hash_map::Entry}, @@ -1292,13 +1295,21 @@ impl BackgroundTask for SwitchPortSettingsManager { } }; - let service_zone_nat_entries = match self + let ( + blueprint_external_networking_generation, + service_zone_nat_entries, + ) = match self .rx_blueprint .borrow_and_update() .clone() - .map(|bp| bp.blueprint.to_service_zone_nat_entries()) + .map(|bp| bp.blueprint.to_service_zone_nat_entries().map( + |entries| ( + bp.blueprint.external_networking_generation, + entries, + ) + )) { - Some(Ok(entries)) => entries, + Some(Ok((generation, entries))) => (generation, entries), Some(Err(err)) => { error!( log, @@ -1314,7 +1325,12 @@ impl BackgroundTask for SwitchPortSettingsManager { } }; - let desired_config = SystemNetworkingConfig { + // The construction here is slightly weird - we start with + // `blueprint_external_networking_config: None` and then + // immediately fill it in. This gives us a non-optional + // reference to the config we supplied, which we need below to + // call `does_bootstore_need_update()`. + let mut desired_config = SystemNetworkingConfig { rack_network_config: RackNetworkConfig { rack_subnet: subnet, infra_ip_first, @@ -1323,8 +1339,16 @@ impl BackgroundTask for SwitchPortSettingsManager { bgp, bfd, }, - service_zone_nat_entries: Some(service_zone_nat_entries), + blueprint_external_networking_config: None, }; + let desired_blueprint_networking_config = &*desired_config + .blueprint_external_networking_config + .insert( + BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation, + service_zone_nat_entries, + }, + ); // bootstore_needs_update is a boolean value that determines // whether or not we need to increment the bootstore version and @@ -1353,37 +1377,12 @@ impl BackgroundTask for SwitchPortSettingsManager { .and_then(|envelope| envelope.deserialize_body()) { Ok(config) => { - let SystemNetworkingConfig { - rack_network_config: current_rnc, - service_zone_nat_entries: current_nat, - } = &config; - let SystemNetworkingConfig { - rack_network_config: desired_rnc, - service_zone_nat_entries: desired_nat, - } = &desired_config; - - let rnc_differs = { - !hashset_eq(current_rnc.bgp.clone(), desired_rnc.bgp.clone()) || - !hashset_eq(current_rnc.bfd.clone(), desired_rnc.bfd.clone()) || - !hashset_eq(current_rnc.ports.clone(), desired_rnc.ports.clone()) || - current_rnc.rack_subnet != desired_rnc.rack_subnet || - current_rnc.infra_ip_first != desired_rnc.infra_ip_first || - current_rnc.infra_ip_last != desired_rnc.infra_ip_last - }; - - let nat_differs = current_nat != desired_nat; - - if rnc_differs || nat_differs { - info!( - log, - "system network config has changed"; - "old" => ?config, - "new" => ?desired_config, - ); - true - } else { - false - } + does_bootstore_need_update( + &config, + &desired_config.rack_network_config, + desired_blueprint_networking_config, + &log, + ) }, Err(e) => { error!( @@ -1536,12 +1535,12 @@ impl BackgroundTask for SwitchPortSettingsManager { } } -fn hashset_eq(left: Vec, right: Vec) -> bool +fn hashset_eq(left: &[T], right: &[T]) -> bool where T: Hash + Eq, { - let left = left.into_iter().collect::>(); - let right = right.into_iter().collect::>(); + let left = left.iter().collect::>(); + let right = right.iter().collect::>(); left == right } @@ -2403,3 +2402,381 @@ async fn add_static_routes( }; } } + +fn does_bootstore_need_update( + current_contents: &SystemNetworkingConfig, + desired_rack_network_config: &RackNetworkConfig, + desired_blueprint_networking_config: &BlueprintExternalNetworkingConfig, + log: &slog::Logger, +) -> bool { + // If `current_contents` don't have a blueprint networking config at all, we + // definitely need to send an update. + let Some(current_blueprint_networking_config) = + current_contents.blueprint_external_networking_config.as_ref() + else { + info!( + log, + "will update bootstore: current contents has no blueprint config", + ); + // TODO-correctness See the note below about not returning early here if + // the rack network config is stale, once we have a way to tell. + return true; + }; + + // Check whether we need to definitely or definitely not perform an update + // based on the blueprint configs: + // + // * "Definitely not" if our generation is older than the generation + // currently in the bootstore; this indicates we've produced + // `desired_blueprint_networking_config` based on a stale blueprint. + // * "Definitely yes" if our generation is not older than the current + // bootstore generation and there have been changes to the config. + // * "No information" if our NAT entries haven't changed, we fall through to + // checking the rack network config below. + // + // If our generation exactly matches the bootstore generation, we expect + // that the NAT entries will also match. We don't explicitly check for this + // here. There's one case where we might encounter "same generation, + // different NAT entries" in practice: if the generation is exactly 1, we've + // just transitioned to a world where we're tracking this generation at all. + // It's possible the most-recently-written config in the bootstore was based + // on a stale blueprint, so we'd see a current blueprint with generation 1 + // and different NAT entries. In this case we'll overwrite the bootstore + // with the correct generation 1 value; any subsequent config changes will + // bump us beyond generation 1. + { + let BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: current_gen, + service_zone_nat_entries: current_nat, + } = current_blueprint_networking_config; + + let BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: desired_gen, + service_zone_nat_entries: desired_nat, + } = desired_blueprint_networking_config; + + // This check must be "strictly less than", not "<=". It's very possible + // the blueprint config has not changed (i.e., we'd expect equal + // generation numbers) but the rack network config (checked below) has. + // We only bail out here if we know we must not send this update. + if desired_gen < current_gen { + warn!( + log, + "skipping bootstore update due to stale blueprint"; + "our-blueprint-gen" => desired_gen, + "bootstore-blueprint-gen" => current_gen, + ); + return false; + } + + // Our blueprint isn't stale - do we need an update for the NAT entries? + if current_nat != desired_nat { + info!( + log, + "will update bootstore: service NAT entries have changed"; + ); + // TODO-correctness See the note below about not returning early + // here if the rack network config is stale, once we have a way to + // tell. + return true; + } + } + + // Check whether we need to perform an update based on the rack network + // configs. + // + // TODO-correctness This needs a similar generation guard as the blueprint + // checks above! As written, if the config is different, we always perform + // an update, even if the config is different because we have old data and + // are comparing against a bootstore written by another Nexus with fresher + // data. Once this is fixed, we'll also need to update the early `return + // true`s above; needing an update for the blueprint config shouldn't + // override our desire to avoid stomping on a newer rack network config. + { + let RackNetworkConfig { + rack_subnet: current_subnet, + infra_ip_first: current_infra_ip_first, + infra_ip_last: current_infra_ip_last, + ports: current_ports, + bgp: current_bgp, + bfd: current_bfd, + } = ¤t_contents.rack_network_config; + + let RackNetworkConfig { + rack_subnet: desired_subnet, + infra_ip_first: desired_infra_ip_first, + infra_ip_last: desired_infra_ip_last, + ports: desired_ports, + bgp: desired_bgp, + bfd: desired_bfd, + } = desired_rack_network_config; + + let rnc_differs = !hashset_eq(current_bgp, desired_bgp) + || !hashset_eq(current_bfd, desired_bfd) + || !hashset_eq(current_ports, desired_ports) + || current_subnet != desired_subnet + || current_infra_ip_first != desired_infra_ip_first + || current_infra_ip_last != desired_infra_ip_last; + + if rnc_differs { + info!( + log, + "will update bootstore: rack network config has changed"; + "old" => ?current_contents.rack_network_config, + "new" => ?desired_rack_network_config, + ); + } + + rnc_differs + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iddqd::IdOrdMap; + use omicron_common::api::external::Generation; + use omicron_common::api::external::Vni; + use omicron_test_utils::dev::test_setup_log; + use sled_agent_types::inventory::SourceNatConfigGeneric; + use sled_agent_types::system_networking::ServiceZoneNatEntries; + use sled_agent_types::system_networking::ServiceZoneNatEntry; + use sled_agent_types::system_networking::ServiceZoneNatKind; + + fn make_rack_network_config(rack_subnet: &str) -> RackNetworkConfig { + RackNetworkConfig { + rack_subnet: rack_subnet.parse().unwrap(), + infra_ip_first: "172.20.15.21".parse().unwrap(), + infra_ip_last: "172.20.15.22".parse().unwrap(), + ports: vec![], + bgp: vec![], + bfd: vec![], + } + } + + fn make_nat_entries(nexus_external_ip: &str) -> ServiceZoneNatEntries { + ServiceZoneNatEntries::try_from( + [ + ServiceZoneNatEntry { + zone_id: "00000000-0000-0000-0000-000000000001" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:101::1".parse().unwrap(), + nic_mac: "A8:40:25:FF:80:00".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::BoundaryNtp { + snat_cfg: SourceNatConfigGeneric::new( + "172.20.26.1".parse().unwrap(), + 0, + 16383, + ) + .expect("valid snat cfg"), + }, + }, + ServiceZoneNatEntry { + zone_id: "00000000-0000-0000-0000-000000000002" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:102::1".parse().unwrap(), + nic_mac: "A8:40:25:FF:80:01".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::ExternalDns { + external_ip: "172.20.26.2".parse().unwrap(), + }, + }, + ServiceZoneNatEntry { + zone_id: "00000000-0000-0000-0000-000000000003" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:103::1".parse().unwrap(), + nic_mac: "A8:40:25:FF:80:02".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::Nexus { + external_ip: nexus_external_ip.parse().unwrap(), + }, + }, + ] + .into_iter() + .collect::>(), + ) + .expect("valid service zone NAT entries") + } + + fn make_blueprint_config( + generation: u32, + service_zone_nat_entries: ServiceZoneNatEntries, + ) -> BlueprintExternalNetworkingConfig { + BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: Generation::from_u32( + generation, + ), + service_zone_nat_entries, + } + } + + fn make_system_networking_config( + rnc: RackNetworkConfig, + blueprint: Option, + ) -> SystemNetworkingConfig { + SystemNetworkingConfig { + rack_network_config: rnc, + blueprint_external_networking_config: blueprint, + } + } + + #[test] + fn bootstore_update_when_current_has_no_blueprint_config() { + let logctx = test_setup_log( + "bootstore_update_when_current_has_no_blueprint_config", + ); + + let rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let current = make_system_networking_config(rnc.clone(), None); + let desired_blueprint = + make_blueprint_config(1, make_nat_entries("172.20.26.3")); + + assert!(does_bootstore_need_update( + ¤t, + &rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } + + #[test] + fn bootstore_no_update_when_desired_blueprint_is_strictly_older() { + let logctx = test_setup_log( + "bootstore_no_update_when_desired_blueprint_is_strictly_older", + ); + + let rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let current = make_system_networking_config( + rnc.clone(), + Some(make_blueprint_config(5, make_nat_entries("172.20.26.3"))), + ); + + // Intentionally use different NAT entries here; confirm that we do not + // report needing an update because the generation here (2) is stale + // (current is 5). + let desired_blueprint = + make_blueprint_config(2, make_nat_entries("172.20.26.4")); + + assert!(!does_bootstore_need_update( + ¤t, + &rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } + + #[test] + fn bootstore_update_when_desired_blueprint_is_newer_and_nat_differs() { + let logctx = test_setup_log( + "bootstore_update_when_desired_blueprint_is_newer_and_nat_differs", + ); + + let rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let current = make_system_networking_config( + rnc.clone(), + Some(make_blueprint_config(2, make_nat_entries("172.20.26.3"))), + ); + let desired_blueprint = + make_blueprint_config(5, make_nat_entries("172.20.26.4")); + + assert!(does_bootstore_need_update( + ¤t, + &rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } + + // Pins the "just-transitioned to tracking generation" case explicitly + // noted in the comment inside `does_bootstore_need_update()`: at gen=1, + // the bootstore may have been written with stale NAT entries by a Nexus + // that pre-dates this generation field. Equal gens with different NATs + // must still trigger an update so the correct gen=1 value gets written. + // + // With the current implementation the test would still pass with any + // generation (1 isn't special), but we only need to test that we handle + // this case for generation 1. We never expect a blueprint to have different + // NAT entries without bumping the associated generation number. + #[test] + fn bootstore_update_when_blueprints_equal_and_nat_differs_at_gen_1() { + let logctx = test_setup_log( + "bootstore_update_when_blueprints_equal_and_nat_differs_at_gen_1", + ); + + let rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let current = make_system_networking_config( + rnc.clone(), + Some(make_blueprint_config(1, make_nat_entries("172.20.26.3"))), + ); + let desired_blueprint = + make_blueprint_config(1, make_nat_entries("172.20.26.4")); + + assert!(does_bootstore_need_update( + ¤t, + &rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } + + #[test] + fn bootstore_update_when_nat_matches_but_rnc_differs() { + let logctx = + test_setup_log("bootstore_update_when_nat_matches_but_rnc_differs"); + + let nat = make_nat_entries("172.20.26.3"); + let current_rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let desired_rnc = make_rack_network_config("fd00:1122:3344:200::/56"); + let desired_blueprint = make_blueprint_config(3, nat); + + let current = make_system_networking_config( + current_rnc, + Some(desired_blueprint.clone()), + ); + + assert!(does_bootstore_need_update( + ¤t, + &desired_rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } + + #[test] + fn bootstore_no_update_when_everything_matches() { + let logctx = + test_setup_log("bootstore_no_update_when_everything_matches"); + + let rnc = make_rack_network_config("fd00:1122:3344:100::/56"); + let nat = make_nat_entries("172.20.26.3"); + let desired_blueprint = make_blueprint_config(3, nat); + + let current = make_system_networking_config( + rnc.clone(), + Some(desired_blueprint.clone()), + ); + + assert!(!does_bootstore_need_update( + ¤t, + &rnc, + &desired_blueprint, + &logctx.log, + )); + + logctx.cleanup_successful(); + } +} diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index aa9c5cbd268..72d07afaa32 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -923,7 +923,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { rack_subnet: "fd00:1122:3344:0100::/56".parse().unwrap(), }, // TODO-correctness Can we fill this in for tests? - service_zone_nat_entries: None, + blueprint_external_networking_config: None, }, generation: 1, }; diff --git a/openapi/sled-agent/sled-agent-37.0.0-a1f825.json.gitstub b/openapi/sled-agent/sled-agent-37.0.0-a1f825.json.gitstub new file mode 100644 index 00000000000..8352e4453ff --- /dev/null +++ b/openapi/sled-agent/sled-agent-37.0.0-a1f825.json.gitstub @@ -0,0 +1 @@ +71da5e7d49981b8b676a4737e0d93f5765450b80:openapi/sled-agent/sled-agent-37.0.0-a1f825.json diff --git a/openapi/sled-agent/sled-agent-37.0.0-a1f825.json b/openapi/sled-agent/sled-agent-38.0.0-ebd174.json similarity index 99% rename from openapi/sled-agent/sled-agent-37.0.0-a1f825.json rename to openapi/sled-agent/sled-agent-38.0.0-ebd174.json index d479abd971d..c5700346726 100644 --- a/openapi/sled-agent/sled-agent-37.0.0-a1f825.json +++ b/openapi/sled-agent/sled-agent-38.0.0-ebd174.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "37.0.0" + "version": "38.0.0" }, "paths": { "/artifacts": { @@ -3496,6 +3496,32 @@ ], "additionalProperties": false }, + "BlueprintExternalNetworkingConfig": { + "description": "External networking configuration controlled by Reconfigurator via blueprints.", + "type": "object", + "properties": { + "blueprint_external_networking_generation": { + "description": "The current generation number of the blueprint's external networking config.\n\nThis generation number is only bumped when a new blueprint is produced that changes the external networking configuration in some way.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "service_zone_nat_entries": { + "description": "Set of all Omicron service zone NAT entries.", + "allOf": [ + { + "$ref": "#/components/schemas/ServiceZoneNatEntries" + } + ] + } + }, + "required": [ + "blueprint_external_networking_generation", + "service_zone_nat_entries" + ] + }, "Board": { "description": "A VM's mainboard.", "type": "object", @@ -9772,17 +9798,17 @@ "description": "All configuration needed to set up system-level networking.", "type": "object", "properties": { - "rack_network_config": { - "$ref": "#/components/schemas/RackNetworkConfig" - }, - "service_zone_nat_entries": { + "blueprint_external_networking_config": { "nullable": true, - "description": "Set of all Omicron service zone NAT entries.", + "description": "External networking configuration specified by blueprints.", "allOf": [ { - "$ref": "#/components/schemas/ServiceZoneNatEntries" + "$ref": "#/components/schemas/BlueprintExternalNetworkingConfig" } ] + }, + "rack_network_config": { + "$ref": "#/components/schemas/RackNetworkConfig" } }, "required": [ diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index ccb5f6634a4..d49d5880bb9 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-37.0.0-a1f825.json \ No newline at end of file +sled-agent-38.0.0-ebd174.json \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 501ddfc322d..9ebb3aec559 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -21,7 +21,7 @@ use omicron_common::api::internal::{ }; use sled_agent_types_versions::{ latest, v1, v4, v6, v7, v9, v10, v11, v12, v14, v16, v17, v18, v20, v22, - v24, v25, v26, v28, v29, v30, v31, v33, v34, + v24, v25, v26, v28, v29, v30, v31, v33, v34, v38, }; use sled_diagnostics::SledDiagnosticsQueryOutput; use slog_error_chain::InlineErrorChain; @@ -38,6 +38,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (38, BOOTSTORE_SERVICE_NAT_GENERATION), (37, MODIFY_SVC_ENABLED_NOT_ONLINE_STATE), (36, DROPSHOT_FREEFORM_BODY_DESC), (35, INLINE_ROUTER_PEER_IP_ADDR), @@ -936,7 +937,20 @@ pub trait SledAgentApi { #[endpoint { method = PUT, path = "/network-bootstore-config", - versions = VERSION_BOOTSTORE_SERVICE_NAT.., + versions = VERSION_BOOTSTORE_SERVICE_NAT_GENERATION.., + operation_id = "write_network_bootstore_config", + }] + async fn write_network_bootstore_config_v38( + rqctx: RequestContext, + body: TypedBody, + ) -> Result; + + // As described above, this must not forward to newer versions; sled-agent + // must implement this by faithfully serializing the requested version. + #[endpoint { + method = PUT, + path = "/network-bootstore-config", + versions = VERSION_BOOTSTORE_SERVICE_NAT..VERSION_BOOTSTORE_SERVICE_NAT_GENERATION, operation_id = "write_network_bootstore_config", }] async fn write_network_bootstore_config_v33( diff --git a/sled-agent/rack-setup/src/service.rs b/sled-agent/rack-setup/src/service.rs index f35bcd6734a..ac870af2853 100644 --- a/sled-agent/rack-setup/src/service.rs +++ b/sled-agent/rack-setup/src/service.rs @@ -120,6 +120,7 @@ use sled_agent_types::inventory::{ }; use sled_agent_types::rack_init::rack_init_bootstore_generation; use sled_agent_types::sled::StartSledAgentRequest; +use sled_agent_types::system_networking::BlueprintExternalNetworkingConfig; use sled_agent_types::system_networking::ServiceZoneNatEntriesError; use sled_agent_types::system_networking::SystemNetworkingConfig; use sled_hardware_types::BaseboardId; @@ -1354,7 +1355,7 @@ impl ServiceInner { // TODO-correctness could we wait to put this into the bootstore // until after the service plan is created, once we've finished // moving all system networking into scrimlet reconcilers? - service_zone_nat_entries: None, + blueprint_external_networking_config: None, }; info!(self.log, "Writing initial network configuration to bootstore"); rss_step.update(RssStep::InitialNetworkConfigUpdate); @@ -1406,12 +1407,14 @@ impl ServiceInner { .map_err(SetupServiceError::ConvertPlanToBlueprint)?; // Now that we have a service plan (and therefore a blueprint), we can - // fill in the service_zone_nat_entries in the bootstore. - system_networking_config.service_zone_nat_entries = Some( - blueprint - .to_service_zone_nat_entries() - .map_err(SetupServiceError::InvalidServiceZoneNatEntries)?, - ); + // fill in the `blueprint_external_networking_config` in the bootstore. + system_networking_config.blueprint_external_networking_config = + Some(BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: Generation::new(), + service_zone_nat_entries: blueprint + .to_service_zone_nat_entries() + .map_err(SetupServiceError::InvalidServiceZoneNatEntries)?, + }); info!( self.log, "Writing final system networking configuration to bootstore", diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2eb525d3e53..d3564d48ad3 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -79,7 +79,7 @@ use trust_quorum_types::messages::{ use trust_quorum_types::status::{CommitStatus, CoordinatorStatus, NodeStatus}; // Fixed identifiers for prior versions only -use sled_agent_types_versions::{v1, v20, v25, v26, v30, v33}; +use sled_agent_types_versions::{v1, v20, v25, v26, v30, v33, v38}; use sled_diagnostics::{ SledDiagnosticsCommandHttpOutput, SledDiagnosticsQueryOutput, }; @@ -962,6 +962,7 @@ impl SledAgentApi for SledAgentImpl { use v20::early_networking::EarlyNetworkConfigBody as BodyV20; use v26::early_networking::EarlyNetworkConfigBody as BodyV26; use v30::early_networking::EarlyNetworkConfigBody as BodyV30; + use v33::system_networking::SystemNetworkingConfig as BodyV33; type LatestEnvelope = EarlyNetworkConfigEnvelope; let sa = rqctx.context(); @@ -989,7 +990,7 @@ impl SledAgentApi for SledAgentImpl { )) })?; let body = BodyV20::from(BodyV26::from(BodyV30::from( - latest_version_body, + BodyV33::from(latest_version_body), ))); v20::early_networking::EarlyNetworkConfig { generation: config.generation, @@ -1010,6 +1011,26 @@ impl SledAgentApi for SledAgentImpl { .await } + async fn write_network_bootstore_config_v38( + rqctx: RequestContext, + body: TypedBody, + ) -> Result { + let sa = rqctx.context(); + let bs = sa.bootstore(); + let body = body.into_inner(); + let config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + + bs.update_network_config(config).await.map_err(|e| { + HttpError::for_internal_error(format!( + "failed to write updated config to boot store: {}", + InlineErrorChain::new(&e), + )) + })?; + + Ok(HttpResponseUpdatedNoContent()) + } + async fn write_network_bootstore_config_v33( rqctx: RequestContext, body: TypedBody, diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 8e65a9a4574..04987ab9bdc 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -89,6 +89,7 @@ use sled_agent_types_versions::v25; use sled_agent_types_versions::v26; use sled_agent_types_versions::v30; use sled_agent_types_versions::v33; +use sled_agent_types_versions::v38; use sled_diagnostics::SledDiagnosticsQueryOutput; use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; @@ -408,6 +409,7 @@ impl SledAgentApi for SledAgentSimImpl { use v20::early_networking::EarlyNetworkConfigBody as BodyV20; use v26::early_networking::EarlyNetworkConfigBody as BodyV26; use v30::early_networking::EarlyNetworkConfigBody as BodyV30; + use v33::system_networking::SystemNetworkingConfig as BodyV33; let config = rqctx.context().bootstore_network_config.lock().unwrap().clone(); @@ -430,8 +432,9 @@ impl SledAgentApi for SledAgentSimImpl { // Downconvert from the current version to the v20 version we have to // return from this endpoint. - let body = - BodyV20::from(BodyV26::from(BodyV30::from(latest_version_body))); + let body = BodyV20::from(BodyV26::from(BodyV30::from(BodyV33::from( + latest_version_body, + )))); Ok(HttpResponseOk(v20::early_networking::EarlyNetworkConfig { generation: config.generation, @@ -440,6 +443,19 @@ impl SledAgentApi for SledAgentSimImpl { })) } + async fn write_network_bootstore_config_v38( + rqctx: RequestContext, + body: TypedBody, + ) -> Result { + let mut config = + rqctx.context().bootstore_network_config.lock().unwrap(); + let body = body.into_inner(); + + *config = EarlyNetworkConfigEnvelope::from(&body.body) + .serialize_to_bootstore_with_generation(body.generation); + Ok(HttpResponseUpdatedNoContent()) + } + async fn write_network_bootstore_config_v33( rqctx: RequestContext, body: TypedBody, diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 384a1ed6ea7..d92a837a403 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -154,7 +154,7 @@ impl SledAgent { }, // TODO-correctness Can we fill this in for the simulated // sled-agent? - service_zone_nat_entries: None, + blueprint_external_networking_config: None, }) .serialize_to_bootstore_with_generation(0), ); diff --git a/sled-agent/tests/data/early_network_blobs.txt b/sled-agent/tests/data/early_network_blobs.txt index 7c14bd091fd..9ff94fcc7b7 100644 --- a/sled-agent/tests/data/early_network_blobs.txt +++ b/sled-agent/tests/data/early_network_blobs.txt @@ -3,3 +3,4 @@ 2026-02-27 pre-r19,{"schema_version":3,"body":{"rack_network_config":{"bfd":[],"bgp":[{"asn":65002,"checker":null,"max_paths":1,"originate":["172.20.52.0/22","172.20.26.0/24"],"shaper":null}],"infra_ip_first":"172.20.15.21","infra_ip_last":"172.20.15.22","ports":[{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":"172.20.15.53/29","vlan_id":null}],"autoneg":false,"bgp_peers":[{"addr":"172.20.15.51","allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":3,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"router_lifetime":0,"vlan_id":null}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":"172.20.15.45/29","vlan_id":null}],"autoneg":false,"bgp_peers":[{"addr":"172.20.15.43","allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":0,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"router_lifetime":0,"vlan_id":null}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"}],"rack_subnet":"fd00:1122:3344:100::/56"}}} 2026-03-17 pre-r19,{"schema_version":4,"body":{"rack_network_config":{"bfd":[],"bgp":[{"asn":65002,"checker":null,"max_paths":1,"originate":["172.20.52.0/22","172.20.26.0/24"],"shaper":null}],"infra_ip_first":"172.20.15.21","infra_ip_last":"172.20.15.22","ports":[{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"type":"addrconf"},"vlan_id":1}],"autoneg":false,"bgp_peers":[{"addr":{"router_lifetime":1234,"type":"unnumbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":3,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":1}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"ip_net":"172.20.15.45/29","type":"static"},"vlan_id":null}],"autoneg":false,"bgp_peers":[{"addr":{"ip":"172.20.15.43","type":"numbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":0,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":null}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"}],"rack_subnet":"fd00:1122:3344:100::/56"}}} 2026-04-01 pre-r19,{"schema_version":5,"body":{"rack_network_config":{"bfd":[],"bgp":[{"asn":65002,"checker":null,"max_paths":1,"originate":["172.20.52.0/22","172.20.26.0/24"],"shaper":null}],"infra_ip_first":"172.20.15.21","infra_ip_last":"172.20.15.22","ports":[{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"type":"addrconf"},"vlan_id":1}],"autoneg":false,"bgp_peers":[{"addr":{"router_lifetime":1234,"type":"unnumbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":3,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":1}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"ip_net":"172.20.15.45/29","type":"static"},"vlan_id":null}],"autoneg":false,"bgp_peers":[{"addr":{"ip":"172.20.15.43","type":"numbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":0,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":null}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"}],"rack_subnet":"fd00:1122:3344:100::/56"},"service_zone_nat_entries":[{"kind":{"external_ip":"172.20.26.6","kind":"nexus"},"nic_mac":"A8:40:25:FF:80:04","sled_underlay_ip":"fd00:1122:3344:102::1","vni":100,"zone_id":"03ee5ea0-a003-4ff3-9125-bf54d41b1868"},{"kind":{"kind":"boundary_ntp","snat_cfg":{"first_port":16384,"ip":"172.20.26.7","last_port":32767}},"nic_mac":"A8:40:25:FF:80:02","sled_underlay_ip":"fd00:1122:3344:101::1","vni":100,"zone_id":"1683d46d-69c4-4adb-a113-70eba32de76f"},{"kind":{"external_ip":"172.20.26.1","kind":"external_dns"},"nic_mac":"A8:40:25:FF:80:03","sled_underlay_ip":"fd00:1122:3344:102::1","vni":100,"zone_id":"45aa654b-77b9-4f73-b0e0-fbf1be4bf30f"},{"kind":{"external_ip":"172.20.26.2","kind":"external_dns"},"nic_mac":"A8:40:25:FF:80:01","sled_underlay_ip":"fd00:1122:3344:105::1","vni":100,"zone_id":"7d6c20e7-92ca-46b9-8ec2-9c003d05cc83"},{"kind":{"external_ip":"172.20.26.8","kind":"nexus"},"nic_mac":"A8:40:25:FF:80:05","sled_underlay_ip":"fd00:1122:3344:108::1","vni":100,"zone_id":"84be6867-c3b1-4f54-92c8-1ba3390a9ff7"},{"kind":{"kind":"boundary_ntp","snat_cfg":{"first_port":0,"ip":"172.20.26.7","last_port":16383}},"nic_mac":"A8:40:25:FF:80:00","sled_underlay_ip":"fd00:1122:3344:103::1","vni":100,"zone_id":"b922e5ec-a05e-4d8a-8378-5277f19426bc"}]}} +2026-04-28 pre-r20,{"schema_version":6,"body":{"blueprint_external_networking_config":{"blueprint_external_networking_generation":1,"service_zone_nat_entries":[{"kind":{"external_ip":"172.20.26.6","kind":"nexus"},"nic_mac":"A8:40:25:FF:80:04","sled_underlay_ip":"fd00:1122:3344:102::1","vni":100,"zone_id":"03ee5ea0-a003-4ff3-9125-bf54d41b1868"},{"kind":{"kind":"boundary_ntp","snat_cfg":{"first_port":16384,"ip":"172.20.26.7","last_port":32767}},"nic_mac":"A8:40:25:FF:80:02","sled_underlay_ip":"fd00:1122:3344:101::1","vni":100,"zone_id":"1683d46d-69c4-4adb-a113-70eba32de76f"},{"kind":{"external_ip":"172.20.26.1","kind":"external_dns"},"nic_mac":"A8:40:25:FF:80:03","sled_underlay_ip":"fd00:1122:3344:102::1","vni":100,"zone_id":"45aa654b-77b9-4f73-b0e0-fbf1be4bf30f"},{"kind":{"external_ip":"172.20.26.2","kind":"external_dns"},"nic_mac":"A8:40:25:FF:80:01","sled_underlay_ip":"fd00:1122:3344:105::1","vni":100,"zone_id":"7d6c20e7-92ca-46b9-8ec2-9c003d05cc83"},{"kind":{"external_ip":"172.20.26.8","kind":"nexus"},"nic_mac":"A8:40:25:FF:80:05","sled_underlay_ip":"fd00:1122:3344:108::1","vni":100,"zone_id":"84be6867-c3b1-4f54-92c8-1ba3390a9ff7"},{"kind":{"kind":"boundary_ntp","snat_cfg":{"first_port":0,"ip":"172.20.26.7","last_port":16383}},"nic_mac":"A8:40:25:FF:80:00","sled_underlay_ip":"fd00:1122:3344:103::1","vni":100,"zone_id":"b922e5ec-a05e-4d8a-8378-5277f19426bc"}]},"rack_network_config":{"bfd":[],"bgp":[{"asn":65002,"checker":null,"max_paths":1,"originate":["172.20.52.0/22","172.20.26.0/24"],"shaper":null}],"infra_ip_first":"172.20.15.21","infra_ip_last":"172.20.15.22","ports":[{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"type":"addrconf"},"vlan_id":1}],"autoneg":false,"bgp_peers":[{"addr":{"router_lifetime":1234,"type":"unnumbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":3,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":1}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch1","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[{"address":{"ip_net":"172.20.15.45/29","type":"static"},"vlan_id":null}],"autoneg":false,"bgp_peers":[{"addr":{"ip":"172.20.15.43","type":"numbered"},"allowed_export":{"type":"allow","value":["172.20.52.0/22","172.20.26.0/24"]},"allowed_import":{"type":"no_filtering"},"asn":65002,"communities":[],"connect_retry":3,"delay_open":3,"enforce_first_as":false,"hold_time":6,"idle_hold_time":0,"keepalive":2,"local_pref":null,"md5_auth_key":null,"min_ttl":null,"multi_exit_discriminator":null,"port":"qsfp18","remote_asn":null,"vlan_id":null}],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp18","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":null,"port":"qsfp0","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":null,"uplink_port_speed":"speed100_g"},{"addresses":[],"autoneg":false,"bgp_peers":[],"lldp":{"chassis_id":null,"management_addrs":null,"port_description":null,"port_id":null,"status":"disabled","system_description":null,"system_name":null},"port":"qsfp26","routes":[],"switch":"switch0","tx_eq":null,"uplink_port_fec":"rs","uplink_port_speed":"speed100_g"}],"rack_subnet":"fd00:1122:3344:100::/56"}}} diff --git a/sled-agent/tests/integration_tests/early_network.rs b/sled-agent/tests/integration_tests/early_network.rs index 5a0abc137c8..f563e6038ec 100644 --- a/sled-agent/tests/integration_tests/early_network.rs +++ b/sled-agent/tests/integration_tests/early_network.rs @@ -6,7 +6,7 @@ use bootstore::schemes::v0 as bootstore; use iddqd::IdOrdMap; -use omicron_common::api::external::Vni; +use omicron_common::api::external::{Generation, Vni}; use omicron_test_utils::dev::test_setup_log; use sled_agent_types::early_networking::{ BgpConfig, BgpPeerConfig, EarlyNetworkConfigEnvelope, ImportExportPolicy, @@ -16,8 +16,8 @@ use sled_agent_types::early_networking::{ }; use sled_agent_types::inventory::SourceNatConfigGeneric; use sled_agent_types::system_networking::{ - ServiceZoneNatEntries, ServiceZoneNatEntry, ServiceZoneNatKind, - SystemNetworkingConfig, + BlueprintExternalNetworkingConfig, ServiceZoneNatEntries, + ServiceZoneNatEntry, ServiceZoneNatKind, SystemNetworkingConfig, }; use slog_error_chain::InlineErrorChain; @@ -133,7 +133,7 @@ fn early_network_blobs_deserialize() { /// future, older blobs can still be deserialized correctly. fn current_config_example() -> (&'static str, EarlyNetworkConfigEnvelope) { // NOTE: the description must not contain commas or newlines. - let description = "2026-04-01 pre-r19"; + let description = "2026-04-28 pre-r20"; let config = EarlyNetworkConfigEnvelope::from(&SystemNetworkingConfig { rack_network_config: RackNetworkConfig { rack_subnet: "fd00:1122:3344:100::/56".parse().unwrap(), @@ -311,102 +311,105 @@ fn current_config_example() -> (&'static str, EarlyNetworkConfigEnvelope) { }], bfd: vec![], }, - service_zone_nat_entries: Some( - ServiceZoneNatEntries::try_from( - [ - ServiceZoneNatEntry { - zone_id: "b922e5ec-a05e-4d8a-8378-5277f19426bc" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:103::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:00".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::BoundaryNtp { - snat_cfg: SourceNatConfigGeneric::new( - "172.20.26.7".parse().unwrap(), - 0, - 16383, - ) - .expect("valid snat cfg"), + blueprint_external_networking_config: Some( + BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: Generation::new(), + service_zone_nat_entries: ServiceZoneNatEntries::try_from( + [ + ServiceZoneNatEntry { + zone_id: "b922e5ec-a05e-4d8a-8378-5277f19426bc" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:103::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:00".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::BoundaryNtp { + snat_cfg: SourceNatConfigGeneric::new( + "172.20.26.7".parse().unwrap(), + 0, + 16383, + ) + .expect("valid snat cfg"), + }, }, - }, - ServiceZoneNatEntry { - zone_id: "1683d46d-69c4-4adb-a113-70eba32de76f" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:101::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:02".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::BoundaryNtp { - snat_cfg: SourceNatConfigGeneric::new( - "172.20.26.7".parse().unwrap(), - 16384, - 32767, - ) - .expect("valid snat cfg"), + ServiceZoneNatEntry { + zone_id: "1683d46d-69c4-4adb-a113-70eba32de76f" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:101::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:02".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::BoundaryNtp { + snat_cfg: SourceNatConfigGeneric::new( + "172.20.26.7".parse().unwrap(), + 16384, + 32767, + ) + .expect("valid snat cfg"), + }, }, - }, - ServiceZoneNatEntry { - zone_id: "84be6867-c3b1-4f54-92c8-1ba3390a9ff7" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:108::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:05".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::Nexus { - external_ip: "172.20.26.8".parse().unwrap(), + ServiceZoneNatEntry { + zone_id: "84be6867-c3b1-4f54-92c8-1ba3390a9ff7" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:108::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:05".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::Nexus { + external_ip: "172.20.26.8".parse().unwrap(), + }, }, - }, - ServiceZoneNatEntry { - zone_id: "03ee5ea0-a003-4ff3-9125-bf54d41b1868" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:102::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:04".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::Nexus { - external_ip: "172.20.26.6".parse().unwrap(), + ServiceZoneNatEntry { + zone_id: "03ee5ea0-a003-4ff3-9125-bf54d41b1868" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:102::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:04".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::Nexus { + external_ip: "172.20.26.6".parse().unwrap(), + }, }, - }, - ServiceZoneNatEntry { - zone_id: "45aa654b-77b9-4f73-b0e0-fbf1be4bf30f" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:102::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:03".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::ExternalDns { - external_ip: "172.20.26.1".parse().unwrap(), + ServiceZoneNatEntry { + zone_id: "45aa654b-77b9-4f73-b0e0-fbf1be4bf30f" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:102::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:03".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::ExternalDns { + external_ip: "172.20.26.1".parse().unwrap(), + }, }, - }, - ServiceZoneNatEntry { - zone_id: "7d6c20e7-92ca-46b9-8ec2-9c003d05cc83" - .parse() - .unwrap(), - sled_underlay_ip: "fd00:1122:3344:105::1" - .parse() - .unwrap(), - nic_mac: "A8:40:25:FF:80:01".parse().unwrap(), - vni: Vni::SERVICES_VNI, - kind: ServiceZoneNatKind::ExternalDns { - external_ip: "172.20.26.2".parse().unwrap(), + ServiceZoneNatEntry { + zone_id: "7d6c20e7-92ca-46b9-8ec2-9c003d05cc83" + .parse() + .unwrap(), + sled_underlay_ip: "fd00:1122:3344:105::1" + .parse() + .unwrap(), + nic_mac: "A8:40:25:FF:80:01".parse().unwrap(), + vni: Vni::SERVICES_VNI, + kind: ServiceZoneNatKind::ExternalDns { + external_ip: "172.20.26.2".parse().unwrap(), + }, }, - }, - ] - .into_iter() - .collect::>(), - ) - .expect("valid service zone NAT entries"), + ] + .into_iter() + .collect::>(), + ) + .expect("valid service zone NAT entries"), + }, ), }); diff --git a/sled-agent/types/src/early_networking/serialization.rs b/sled-agent/types/src/early_networking/serialization.rs index 8c04173b14b..21b39c985fc 100644 --- a/sled-agent/types/src/early_networking/serialization.rs +++ b/sled-agent/types/src/early_networking/serialization.rs @@ -50,7 +50,7 @@ use bootstore::schemes::v0 as bootstore; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use sled_agent_types_versions::{latest, v20, v26, v30, v33}; +use sled_agent_types_versions::{latest, v20, v26, v30, v33, v38}; use slog_error_chain::SlogInlineError; #[derive(Debug, thiserror::Error, SlogInlineError)] @@ -305,6 +305,7 @@ impl EarlyNetworkConfigEnvelope { v26::early_networking::EarlyNetworkConfigBody, v30::early_networking::EarlyNetworkConfigBody, v33::system_networking::SystemNetworkingConfig, + v38::system_networking::SystemNetworkingConfig, ); f(self.schema_version, self.body.clone()) } @@ -353,3 +354,4 @@ from_body_for_envelope!(v20::early_networking::EarlyNetworkConfigBody); from_body_for_envelope!(v26::early_networking::EarlyNetworkConfigBody); from_body_for_envelope!(v30::early_networking::EarlyNetworkConfigBody); from_body_for_envelope!(v33::system_networking::SystemNetworkingConfig); +from_body_for_envelope!(v38::system_networking::SystemNetworkingConfig); diff --git a/sled-agent/types/versions/src/bootstore_service_nat_generation/mod.rs b/sled-agent/types/versions/src/bootstore_service_nat_generation/mod.rs new file mode 100644 index 00000000000..8d8f47af7af --- /dev/null +++ b/sled-agent/types/versions/src/bootstore_service_nat_generation/mod.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `BOOTSTORE_SERVICE_NAT_GENERATION` of the Sled Agent API. + +pub mod system_networking; diff --git a/sled-agent/types/versions/src/bootstore_service_nat_generation/system_networking.rs b/sled-agent/types/versions/src/bootstore_service_nat_generation/system_networking.rs new file mode 100644 index 00000000000..8ddb343c3f0 --- /dev/null +++ b/sled-agent/types/versions/src/bootstore_service_nat_generation/system_networking.rs @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types for system-level networking. +//! +//! Changes in this version: +//! +//! * New types added: +//! * [`BlueprintExternalNetworkingConfig`], which wraps a +//! [`ServiceZoneNatEntries`] and a generation number from the blueprint. +//! * Old types redefined: +//! * [`SystemNetworkingConfig`] loses the `service_zone_nat_entries` field it +//! had in favor of the new +//! [`SystemNetworkingConfig::blueprint_external_networking_config`] field. + +use crate::v30::early_networking::RackNetworkConfig; +use crate::v33; +use crate::v33::system_networking::ServiceZoneNatEntries; +use omicron_common::api::external::Generation; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// External networking configuration controlled by Reconfigurator via +/// blueprints. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct BlueprintExternalNetworkingConfig { + /// The current generation number of the blueprint's external networking + /// config. + /// + /// This generation number is only bumped when a new blueprint is produced + /// that changes the external networking configuration in some way. + pub blueprint_external_networking_generation: Generation, + + /// Set of all Omicron service zone NAT entries. + pub service_zone_nat_entries: ServiceZoneNatEntries, +} + +/// All configuration needed to set up system-level networking. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct SystemNetworkingConfig { + pub rack_network_config: RackNetworkConfig, + + /// External networking configuration specified by blueprints. + // + // This field is optional for two reasons: + // + // 1. RSS has to initially populate a `SystemNetworkingConfig` with no + // blueprint-based networking config to start all the sled-agents. Once + // they all start, it computes a service plan, at which point it can fill + // this field in. + // 2. Backwards compatibility: prior versions of this type did not store + // this information at all, and we must be able to cleanly handle that at + // runtime. + // + // In the future, if we can find a way to relax RSS, we can eventually make + // this field non-optional (once we're confident all deployed systems are + // past the release we start populating this field). + pub blueprint_external_networking_config: + Option, +} + +impl SystemNetworkingConfig { + pub const SCHEMA_VERSION: u32 = 6; +} + +impl From + for v33::system_networking::SystemNetworkingConfig +{ + fn from(value: SystemNetworkingConfig) -> Self { + Self { + rack_network_config: value.rack_network_config, + service_zone_nat_entries: value + .blueprint_external_networking_config + .map(|config| config.service_zone_nat_entries), + } + } +} + +impl From + for SystemNetworkingConfig +{ + fn from(value: v33::system_networking::SystemNetworkingConfig) -> Self { + Self { + rack_network_config: value.rack_network_config, + blueprint_external_networking_config: value + .service_zone_nat_entries + .map(|service_zone_nat_entries| { + // Any config older than this version is from before when + // the blueprint _had_ an `external_networking_generation`. + // The initial migration set the value to + // `Generation::new()` (i.e., `1`), so we do the same here. + BlueprintExternalNetworkingConfig { + blueprint_external_networking_generation: + Generation::new(), + service_zone_nat_entries, + } + }), + } + } +} + +/// Structure for requests from Nexus to sled-agent to write a new +/// [`SystemNetworkingConfig`] into the replicated bootstore. +/// +/// [`WriteNetworkConfigRequest`] INTENTIONALLY does not have a `From` +/// implementation from prior API versions. It is critically important that +/// sled-agent not attempt to rewrite old [`SystemNetworkingConfig`] types to +/// the latest version. For more about this, see the comments on the relevant +/// endpoint in `sled-agent-api`. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct WriteNetworkConfigRequest { + pub generation: u64, + pub body: SystemNetworkingConfig, +} diff --git a/sled-agent/types/versions/src/latest.rs b/sled-agent/types/versions/src/latest.rs index 7ceb9d668c9..a8c1c37435e 100644 --- a/sled-agent/types/versions/src/latest.rs +++ b/sled-agent/types/versions/src/latest.rs @@ -237,8 +237,10 @@ pub mod system_networking { pub use crate::v33::system_networking::ServiceZoneNatEntriesError; pub use crate::v33::system_networking::ServiceZoneNatEntry; pub use crate::v33::system_networking::ServiceZoneNatKind; - pub use crate::v33::system_networking::SystemNetworkingConfig; - pub use crate::v33::system_networking::WriteNetworkConfigRequest; + + pub use crate::v38::system_networking::BlueprintExternalNetworkingConfig; + pub use crate::v38::system_networking::SystemNetworkingConfig; + pub use crate::v38::system_networking::WriteNetworkConfigRequest; } pub mod trust_quorum { diff --git a/sled-agent/types/versions/src/lib.rs b/sled-agent/types/versions/src/lib.rs index 8b8fc8071c0..c69309449f9 100644 --- a/sled-agent/types/versions/src/lib.rs +++ b/sled-agent/types/versions/src/lib.rs @@ -81,6 +81,8 @@ pub mod v33; pub mod v34; #[path = "modify_svc_enabled_not_online_state/mod.rs"] pub mod v37; +#[path = "bootstore_service_nat_generation/mod.rs"] +pub mod v38; #[path = "add_nexus_lockstep_port_to_inventory/mod.rs"] pub mod v4; #[path = "add_probe_put_endpoint/mod.rs"]