From 82cf6858cf3f4b23ba28bef71c642c17d8c5677b Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 17 Dec 2025 15:41:46 -0500 Subject: [PATCH 1/8] initial: add last_allocated_ip_subnet_offset to blueprint --- Cargo.lock | 2 + common/Cargo.toml | 1 + common/src/address.rs | 5 +++ dev-tools/omdb/tests/successes.out | 4 ++ .../output/cmds-add-sled-no-disks-stdout | 4 ++ .../tests/output/cmds-example-stdout | 7 ++++ ...ds-expunge-newly-added-external-dns-stdout | 9 +++++ ...ds-expunge-newly-added-internal-dns-stdout | 3 ++ .../tests/output/cmds-expunge-zones-stdout | 3 ++ .../output/cmds-host-phase-2-source-stdout | 2 + .../output/cmds-mupdate-update-flow-stdout | 9 +++++ .../cmds-nexus-generation-autobump-stdout | 3 ++ .../tests/output/cmds-set-mgs-updates-stdout | 15 ++++++++ .../cmds-set-remove-mupdate-override-stdout | 14 +++++++ .../tests/output/cmds-set-zone-images-stdout | 3 ++ .../tests/output/cmds-target-release-stdout | 3 ++ .../tests/output/cmds-unsafe-zone-mgs-stdout | 3 ++ nexus/db-model/src/deployment.rs | 1 + .../db-queries/src/db/datastore/deployment.rs | 5 +++ nexus/db-schema/src/schema.rs | 1 + .../reconfigurator/execution/src/database.rs | 2 + nexus/reconfigurator/execution/src/dns.rs | 2 + .../execution/src/omicron_sled_config.rs | 2 + .../src/blueprint_editor/sled_editor.rs | 3 ++ .../sled_editor/underlay_ip_allocator.rs | 31 ++++++++++++++++ .../example_builder_zone_counts_blueprint.txt | 5 +++ .../planner_decommissions_sleds_bp2.txt | 3 ++ .../output/planner_nonprovisionable_bp2.txt | 5 +++ .../background/tasks/blueprint_execution.rs | 3 +- nexus/test-utils/src/starter.rs | 2 + nexus/types/Cargo.toml | 1 + nexus/types/src/deployment.rs | 37 +++++++++++++++++-- .../types/src/deployment/blueprint_display.rs | 1 + schema/crdb/dbinit.sql | 3 ++ sled-agent/src/rack_setup/plan/service.rs | 1 + 35 files changed, 193 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcf081ea019..99c2a6083f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7502,6 +7502,7 @@ dependencies = [ "illumos-utils", "indent_write", "internal-dns-types", + "ipnet", "ipnetwork", "itertools 0.14.0", "newtype-uuid", @@ -8064,6 +8065,7 @@ dependencies = [ "serde_with", "slog", "slog-error-chain", + "static_assertions", "strum 0.27.2", "test-strategy", "thiserror 2.0.17", diff --git a/common/Cargo.toml b/common/Cargo.toml index f62f78b4207..afbaa9d5051 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -44,6 +44,7 @@ serde_json.workspace = true serde_with.workspace = true slog.workspace = true slog-error-chain.workspace = true +static_assertions.workspace = true strum.workspace = true test-strategy = { workspace = true, optional = true } thiserror.workspace = true diff --git a/common/src/address.rs b/common/src/address.rs index 94ff1aa9d43..0e8d18fff5b 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -278,6 +278,11 @@ pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF; // to assume that addresses in this subnet are available. pub const SLED_RESERVED_ADDRESSES: u16 = 32; +static_assertions::const_assert_eq!( + RSS_RESERVED_ADDRESSES, + SLED_RESERVED_ADDRESSES +); + /// Wraps an [`Ipv6Net`] with a compile-time prefix length. #[derive( Debug, diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 2738d6ff1fb..5fffe55f403 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -1556,6 +1556,7 @@ parent: state::::::::::::: active config generation: 2 subnet:::::::::::: ::/64 + last allocated IP: ::20 host phase 2 contents: ------------------------ @@ -1598,6 +1599,7 @@ parent: state::::::::::::: active config generation: 2 subnet:::::::::::: ::/64 + last allocated IP: ::20 host phase 2 contents: ------------------------ @@ -1686,6 +1688,7 @@ parent: state::::::::::::: active config generation: 2 subnet:::::::::::: ::/64 + last allocated IP: ::20 host phase 2 contents: ------------------------ @@ -1728,6 +1731,7 @@ parent: state::::::::::::: active config generation: 2 subnet:::::::::::: ::/64 + last allocated IP: ::20 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout index 0e7584d3ea5..40819e3fb48 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout @@ -61,6 +61,7 @@ parent: dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 state::::::::::::: active config generation: 1 subnet:::::::::::: fd00:1122:3344:104::/64 + last allocated IP: fd00:1122:3344:104::20 host phase 2 contents: ------------------------ @@ -93,6 +94,7 @@ parent: dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::28 host phase 2 contents: ------------------------ @@ -161,6 +163,7 @@ parent: dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::27 host phase 2 contents: ------------------------ @@ -226,6 +229,7 @@ parent: dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index 83aa46e3d5c..396467b7d03 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -86,6 +86,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2f host phase 2 contents: ------------------------ @@ -203,6 +204,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2e host phase 2 contents: ------------------------ @@ -317,6 +319,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -510,6 +513,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2f host phase 2 contents: ------------------------ @@ -1165,6 +1169,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::24 host phase 2 contents: ------------------------ @@ -1220,6 +1225,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::24 host phase 2 contents: ------------------------ @@ -1275,6 +1281,7 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout index 567030c7ba4..920e49b4901 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout @@ -15,6 +15,7 @@ parent: 06c88262-f435-410e-ba98-101bed41ec27 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2f host phase 2 contents: ------------------------ @@ -132,6 +133,7 @@ parent: 06c88262-f435-410e-ba98-101bed41ec27 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2e host phase 2 contents: ------------------------ @@ -246,6 +248,7 @@ parent: 06c88262-f435-410e-ba98-101bed41ec27 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -562,6 +565,7 @@ parent: 3f00b694-1b16-4aaa-8f78-e6b3a527b434 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2f host phase 2 contents: ------------------------ @@ -679,6 +683,7 @@ parent: 3f00b694-1b16-4aaa-8f78-e6b3a527b434 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2e host phase 2 contents: ------------------------ @@ -793,6 +798,7 @@ parent: 3f00b694-1b16-4aaa-8f78-e6b3a527b434 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -1134,6 +1140,7 @@ parent: 366b0b68-d80e-4bc1-abd3-dc69837847e0 state::::::::::::: active config generation: 4 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::30 host phase 2 contents: ------------------------ @@ -1254,6 +1261,7 @@ parent: 366b0b68-d80e-4bc1-abd3-dc69837847e0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2e host phase 2 contents: ------------------------ @@ -1368,6 +1376,7 @@ parent: 366b0b68-d80e-4bc1-abd3-dc69837847e0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout index 8d8f96928c4..03dce4ee63e 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout @@ -13,6 +13,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2f host phase 2 contents: ------------------------ @@ -130,6 +131,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -244,6 +246,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2e host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-zones-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-zones-stdout index 213417a4d10..bf6301d2f6f 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-zones-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-zones-stdout @@ -18,6 +18,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::28 host phase 2 contents: ------------------------ @@ -86,6 +87,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::27 host phase 2 contents: ------------------------ @@ -151,6 +153,7 @@ parent: 184f10b3-61cb-41ef-9b93-3489b2bac559 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-host-phase-2-source-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-host-phase-2-source-stdout index 45933052308..6431e6b6899 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-host-phase-2-source-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-host-phase-2-source-stdout @@ -357,6 +357,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 4 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::35 host phase 2 contents: ------------------------------ @@ -863,6 +864,7 @@ parent: af934083-59b5-4bf6-8966-6fb5292c29e1 state::::::::::::: active config generation: 6 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::35 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 25242992fcd..580c8f9c9fe 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -1225,6 +1225,7 @@ parent: c1a0d242-9160-40f4-96ae-61f8f40a0b1b state::::::::::::: active config generation: 5 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::26 host phase 2 contents: ------------------------ @@ -1279,6 +1280,7 @@ parent: c1a0d242-9160-40f4-96ae-61f8f40a0b1b state::::::::::::: active config generation: 7 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::25 host phase 2 contents: ------------------------ @@ -1330,6 +1332,7 @@ parent: c1a0d242-9160-40f4-96ae-61f8f40a0b1b state::::::::::::: active config generation: 5 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::25 host phase 2 contents: ------------------------ @@ -1565,6 +1568,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state::::::::::::: active config generation: 5 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::26 host phase 2 contents: ------------------------ @@ -1619,6 +1623,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state::::::::::::: active config generation: 7 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::25 host phase 2 contents: ------------------------ @@ -1670,6 +1675,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state::::::::::::: active config generation: 6 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::25 host phase 2 contents: ------------------------ @@ -1940,6 +1946,7 @@ parent: 8f2d1f39-7c88-4701-aa43-56bf281b28c1 state::::::::::::: active config generation: 6 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::26 host phase 2 contents: ------------------------ @@ -1994,6 +2001,7 @@ parent: 8f2d1f39-7c88-4701-aa43-56bf281b28c1 state::::::::::::: active config generation: 7 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::25 host phase 2 contents: ------------------------ @@ -2045,6 +2053,7 @@ parent: 8f2d1f39-7c88-4701-aa43-56bf281b28c1 state::::::::::::: active config generation: 6 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::25 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-nexus-generation-autobump-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-nexus-generation-autobump-stdout index 28743676866..83d802448d1 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-nexus-generation-autobump-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-nexus-generation-autobump-stdout @@ -1077,6 +1077,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::28 host phase 2 contents: ------------------------------ @@ -1145,6 +1146,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::27 host phase 2 contents: ------------------------------ @@ -1210,6 +1212,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-set-mgs-updates-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-set-mgs-updates-stdout index dce677037c7..a2dde8fa118 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-set-mgs-updates-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-set-mgs-updates-stdout @@ -15,6 +15,7 @@ parent: 6ccc786b-17f1-4562-958f-5a7d9a5a15fd state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::28 host phase 2 contents: ------------------------ @@ -83,6 +84,7 @@ parent: 6ccc786b-17f1-4562-958f-5a7d9a5a15fd state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::27 host phase 2 contents: ------------------------ @@ -148,6 +150,7 @@ parent: 6ccc786b-17f1-4562-958f-5a7d9a5a15fd state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ @@ -245,6 +248,7 @@ parent: ad97e762-7bf1-45a6-a98f-60afb7e491c0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::28 host phase 2 contents: ------------------------ @@ -313,6 +317,7 @@ parent: ad97e762-7bf1-45a6-a98f-60afb7e491c0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::27 host phase 2 contents: ------------------------ @@ -378,6 +383,7 @@ parent: ad97e762-7bf1-45a6-a98f-60afb7e491c0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ @@ -557,6 +563,7 @@ parent: cca24b71-09b5-4042-9185-b33e9f2ebba0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::28 host phase 2 contents: ------------------------ @@ -625,6 +632,7 @@ parent: cca24b71-09b5-4042-9185-b33e9f2ebba0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::27 host phase 2 contents: ------------------------ @@ -690,6 +698,7 @@ parent: cca24b71-09b5-4042-9185-b33e9f2ebba0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ @@ -872,6 +881,7 @@ parent: 5bf974f3-81f9-455b-b24e-3099f765664c state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::28 host phase 2 contents: ------------------------ @@ -940,6 +950,7 @@ parent: 5bf974f3-81f9-455b-b24e-3099f765664c state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::27 host phase 2 contents: ------------------------ @@ -1005,6 +1016,7 @@ parent: 5bf974f3-81f9-455b-b24e-3099f765664c state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ @@ -1146,6 +1158,7 @@ parent: 1b837a27-3be1-4fcb-8499-a921c839e1d0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::28 host phase 2 contents: ------------------------ @@ -1214,6 +1227,7 @@ parent: 1b837a27-3be1-4fcb-8499-a921c839e1d0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::27 host phase 2 contents: ------------------------ @@ -1279,6 +1293,7 @@ parent: 1b837a27-3be1-4fcb-8499-a921c839e1d0 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-set-remove-mupdate-override-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-set-remove-mupdate-override-stdout index 38e000249d0..56ea17f7dbb 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-set-remove-mupdate-override-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-set-remove-mupdate-override-stdout @@ -53,6 +53,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::26 host phase 2 contents: ------------------------ @@ -107,6 +108,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::25 host phase 2 contents: ------------------------ @@ -158,6 +160,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:106::/64 + last allocated IP:::::::::::: fd00:1122:3344:106::25 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -210,6 +213,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:104::/64 + last allocated IP:::::::::::: fd00:1122:3344:104::22 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -252,6 +256,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:105::/64 + last allocated IP:::::::::::: fd00:1122:3344:105::22 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -294,6 +299,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:103::/64 + last allocated IP:::::::::::: fd00:1122:3344:103::22 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -336,6 +342,7 @@ parent: df06bb57-ad42-4431-9206-abff322896c7 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:107::/64 + last allocated IP:::::::::::: fd00:1122:3344:107::22 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -720,6 +727,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state:::::::::::::::::::::::: active config generation:::::::::::: 2 subnet::::::::::::::::::::::: fd00:1122:3344:108::/64 + last allocated IP:::::::::::: fd00:1122:3344:108::20 will remove mupdate override: ffffffff-ffff-ffff-ffff-ffffffffffff host phase 2 contents: @@ -753,6 +761,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:102::/64 + last allocated IP:::::::::::: fd00:1122:3344:102::26 will remove mupdate override: ffffffff-ffff-ffff-ffff-ffffffffffff host phase 2 contents: @@ -808,6 +817,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::25 host phase 2 contents: ------------------------ @@ -859,6 +869,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state:::::::::::::::::::::::: active config generation:::::::::::: 4 subnet::::::::::::::::::::::: fd00:1122:3344:106::/64 + last allocated IP:::::::::::: fd00:1122:3344:106::25 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -911,6 +922,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state:::::::::::::::::::::::: active config generation:::::::::::: 3 subnet::::::::::::::::::::::: fd00:1122:3344:104::/64 + last allocated IP:::::::::::: fd00:1122:3344:104::22 will remove mupdate override: 00000000-0000-0000-0000-000000000000 host phase 2 contents: @@ -953,6 +965,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state:::::::::::::::::::::::: active config generation:::::::::::: 4 subnet::::::::::::::::::::::: fd00:1122:3344:105::/64 + last allocated IP:::::::::::: fd00:1122:3344:105::22 will remove mupdate override: ffffffff-ffff-ffff-ffff-ffffffffffff host phase 2 contents: @@ -995,6 +1008,7 @@ parent: afb09faf-a586-4483-9289-04d4f1d8ba23 state::::::::::::: active config generation: 4 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::22 host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-set-zone-images-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-set-zone-images-stdout index 86d9ef94924..ebacf2d4d60 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-set-zone-images-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-set-zone-images-stdout @@ -15,6 +15,7 @@ parent: 1b013011-2062-4b48-b544-a32b23bce83a state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -140,6 +141,7 @@ parent: 9766ca20-38d4-4380-b005-e7c43c797e7c state::::::::::::: active config generation: 4 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ @@ -381,6 +383,7 @@ parent: bb128f06-a2e1-44c1-8874-4f789d0ff896 state::::::::::::: active config generation: 6 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2e host phase 2 contents: ------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 952056fb802..42756982f53 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -9059,6 +9059,7 @@ parent: 05685571-d61f-4754-a2b2-604ea8c45dff state::::::::::::: active config generation: 18 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2c host phase 2 contents: ------------------------------ @@ -9139,6 +9140,7 @@ parent: 05685571-d61f-4754-a2b2-604ea8c45dff state::::::::::::: active config generation: 19 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2c host phase 2 contents: ------------------------------ @@ -9218,6 +9220,7 @@ parent: 05685571-d61f-4754-a2b2-604ea8c45dff state::::::::::::: active config generation: 17 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2b host phase 2 contents: ------------------------------ diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-unsafe-zone-mgs-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-unsafe-zone-mgs-stdout index 82562dc16a7..bcec3ceb38b 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-unsafe-zone-mgs-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-unsafe-zone-mgs-stdout @@ -1086,6 +1086,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::28 host phase 2 contents: ------------------------ @@ -1154,6 +1155,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::27 host phase 2 contents: ------------------------ @@ -1219,6 +1221,7 @@ parent: 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::27 host phase 2 contents: ------------------------ diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index c5d2fedf4f3..55003227fa9 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -223,6 +223,7 @@ pub struct BpSledMetadata { /// Public only for easy of writing queries; consumers should prefer the /// `subnet()` method. pub subnet: IpNetwork, + pub last_allocated_ip_subnet_offset: SqlU16, } impl BpSledMetadata { diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 67c4e162bc5..edd0699b122 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -260,6 +260,9 @@ impl DataStore { .artifact_hash() .map(ArtifactHash), subnet: Ipv6Network::from(sled.subnet).into(), + last_allocated_ip_subnet_offset: sled + .last_allocated_ip_subnet_offset + .into(), }) .collect::>(); @@ -800,6 +803,8 @@ impl DataStore { state: s.sled_state.into(), subnet, sled_agent_generation: *s.sled_agent_generation, + last_allocated_ip_subnet_offset: *s + .last_allocated_ip_subnet_offset, disks: IdOrdMap::new(), datasets: IdOrdMap::new(), zones: IdOrdMap::new(), diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 3ef6efb4b0a..7778da87ca6 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -2057,6 +2057,7 @@ table! { host_phase_2_desired_slot_b -> Nullable, subnet -> Inet, + last_allocated_ip_subnet_offset -> Int4, } } diff --git a/nexus/reconfigurator/execution/src/database.rs b/nexus/reconfigurator/execution/src/database.rs index df96d8bab3b..4a5cfece456 100644 --- a/nexus/reconfigurator/execution/src/database.rs +++ b/nexus/reconfigurator/execution/src/database.rs @@ -87,6 +87,7 @@ mod test { use nexus_types::inventory::NetworkInterface; use nexus_types::inventory::NetworkInterfaceKind; use omicron_common::address::Ipv6Subnet; + use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; @@ -165,6 +166,7 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation: Generation::new(), zones, disks: IdOrdMap::new(), diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 202188c2621..b64c36700b3 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -368,6 +368,7 @@ mod test { use omicron_common::address::Ipv6Subnet; use omicron_common::address::RACK_PREFIX; use omicron_common::address::REPO_DEPOT_PORT; + use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::address::SLED_PREFIX; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; @@ -695,6 +696,7 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(*sa.sled_agent_address.ip()), + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation: ledgered_sled_config.generation, disks: IdOrdMap::new(), datasets: IdOrdMap::new(), diff --git a/nexus/reconfigurator/execution/src/omicron_sled_config.rs b/nexus/reconfigurator/execution/src/omicron_sled_config.rs index 653302dc9d1..566b0752689 100644 --- a/nexus/reconfigurator/execution/src/omicron_sled_config.rs +++ b/nexus/reconfigurator/execution/src/omicron_sled_config.rs @@ -97,6 +97,7 @@ mod tests { use nexus_types::external_api::views::SledState; use omicron_common::address::Ipv6Subnet; use omicron_common::address::REPO_DEPOT_PORT; + use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::CompressionAlgorithm; @@ -262,6 +263,7 @@ mod tests { let sled_config = BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation: sim_sled_agent_config_generation.next(), disks, datasets, diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs index 6c11d42d8d3..0ee71c1da21 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs @@ -579,6 +579,9 @@ impl ActiveSledEditor { config: BlueprintSledConfig { state: SledState::Active, subnet: self.underlay_ip_allocator.subnet(), + last_allocated_ip_subnet_offset: self + .underlay_ip_allocator + .last_allocated_ip_subnet_offset(), sled_agent_generation, disks, datasets, diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs index 332746dd6a2..0b8bfced44a 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs @@ -5,6 +5,7 @@ //! Allocator for zone underlay IP addresses with a single sled's subnet. use ipnet::IpAdd; +use ipnet::IpSub; use omicron_common::address::CP_SERVICES_RESERVED_ADDRESSES; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; @@ -71,6 +72,36 @@ impl SledUnderlayIpAllocator { self.subnet } + /// Get the last allocated IP as an offset into the sled subnet. + pub fn last_allocated_ip_subnet_offset(&self) -> u16 { + let last_allocated_ip = self.last; + let offset = self.last.saturating_sub(self.subnet.net().prefix()); + + // Based on the asserts made in `new()` and the error checking performed + // in `alloc()`, we know `self.last` must be in the range + // `[SLED_RESERVED_ADDRESSES, CP_SERVICES_RESERVED_ADDRESSES]` and + // therefore must fit in a u16. + let offset = match u16::try_from(offset) { + Ok(offset) => offset, + Err(_) => { + unreachable!( + "last allocated ip ({last_allocated_ip}) is beyond \ + the range of expected allocations (offset = {offset})" + ); + } + }; + assert!( + offset >= SLED_RESERVED_ADDRESSES, + "offset unexpectedly inside reserved range: {offset}" + ); + assert!( + offset <= CP_SERVICES_RESERVED_ADDRESSES, + "offset unexpectedly above reserved range: {offset}" + ); + + offset + } + /// Mark an address as used. /// /// Marking an address that has already been handed out by this allocator diff --git a/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt b/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt index 585009ab08c..a760849f2b3 100644 --- a/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt +++ b/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt @@ -5,6 +5,7 @@ parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:105::/64 + last allocated IP: fd00:1122:3344:105::31 host phase 2 contents: ------------------------ @@ -127,6 +128,7 @@ parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:104::/64 + last allocated IP: fd00:1122:3344:104::2f host phase 2 contents: ------------------------ @@ -244,6 +246,7 @@ parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2f host phase 2 contents: ------------------------ @@ -358,6 +361,7 @@ parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2f host phase 2 contents: ------------------------ @@ -472,6 +476,7 @@ parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2f host phase 2 contents: ------------------------ diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index aba7d87fb37..30dd53403b2 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -5,6 +5,7 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd state::::::::::::: decommissioned config generation: 3 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2e host phase 2 contents: ------------------------ @@ -119,6 +120,7 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2f host phase 2 contents: ------------------------ @@ -234,6 +236,7 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2e host phase 2 contents: ------------------------ diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 109098acfa0..e6b533b5ee4 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -5,6 +5,7 @@ parent: 2c4d059d-6498-49ca-8052-6e696a675f6e state::::::::::::: active config generation: 2 subnet:::::::::::: fd00:1122:3344:105::/64 + last allocated IP: fd00:1122:3344:105::2e host phase 2 contents: ------------------------ @@ -119,6 +120,7 @@ parent: 2c4d059d-6498-49ca-8052-6e696a675f6e state::::::::::::: decommissioned config generation: 3 subnet:::::::::::: fd00:1122:3344:103::/64 + last allocated IP: fd00:1122:3344:103::2d host phase 2 contents: ------------------------ @@ -230,6 +232,7 @@ parent: 2c4d059d-6498-49ca-8052-6e696a675f6e state::::::::::::: decommissioned config generation: 3 subnet:::::::::::: fd00:1122:3344:102::/64 + last allocated IP: fd00:1122:3344:102::2d host phase 2 contents: ------------------------ @@ -341,6 +344,7 @@ parent: 2c4d059d-6498-49ca-8052-6e696a675f6e state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:104::/64 + last allocated IP: fd00:1122:3344:104::2f host phase 2 contents: ------------------------ @@ -453,6 +457,7 @@ parent: 2c4d059d-6498-49ca-8052-6e696a675f6e state::::::::::::: active config generation: 3 subnet:::::::::::: fd00:1122:3344:101::/64 + last allocated IP: fd00:1122:3344:101::2f host phase 2 contents: ------------------------ diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index be6c31a889c..26b5433709a 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -247,7 +247,7 @@ mod test { blueprint_zone_type, }; use nexus_types::external_api::views::SledState; - use omicron_common::address::Ipv6Subnet; + use omicron_common::address::{Ipv6Subnet, RSS_RESERVED_ADDRESSES}; use omicron_common::api::external; use omicron_common::api::external::Generation; use omicron_common::zpool_name::ZpoolName; @@ -285,6 +285,7 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation: Generation::new().next(), disks: IdOrdMap::new(), datasets: IdOrdMap::new(), diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index d045fe79671..f8bd484d887 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -61,6 +61,7 @@ use omicron_common::address::Ipv6Subnet; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_PORT; +use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::api::external::Name; @@ -1389,6 +1390,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation, disks, datasets, diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index 973f7d0993f..d68b901c384 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -27,6 +27,7 @@ humantime.workspace = true iddqd.workspace = true illumos-utils.workspace = true indent_write.workspace = true +ipnet.workspace = true ipnetwork.workspace = true itertools.workspace = true newtype_derive.workspace = true diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 080b1f3c883..be0fd3784e4 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -28,6 +28,7 @@ use iddqd::IdOrdMap; use iddqd::id_ord_map::Entry; use iddqd::id_ord_map::RefMut; use iddqd::id_upcast; +use ipnet::IpAdd; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::ByteCount; @@ -939,9 +940,13 @@ impl fmt::Display for BlueprintDisplay<'_> { // Loop through all sleds and print details of their configs. for (sled_id, config) in sleds { + let last_allocated_ip = config.last_allocated_ip(); + let BlueprintSledConfig { state, subnet, + // covered by `last_allocated_ip` computed above + last_allocated_ip_subnet_offset: _, sled_agent_generation, disks, datasets, @@ -952,10 +957,12 @@ impl fmt::Display for BlueprintDisplay<'_> { // Report toplevel sled info writeln!(f, "\n sled: {sled_id}")?; - let mut rows = Vec::new(); - rows.push((STATE, state.to_string())); - rows.push((CONFIG_GENERATION, sled_agent_generation.to_string())); - rows.push((SUBNET, subnet.to_string())); + let mut rows = vec![ + (STATE, state.to_string()), + (CONFIG_GENERATION, sled_agent_generation.to_string()), + (SUBNET, subnet.to_string()), + (LAST_ALLOCATED_IP, last_allocated_ip.to_string()), + ]; if let Some(id) = remove_mupdate_override { rows.push((WILL_REMOVE_MUPDATE_OVERRIDE, id.to_string())); @@ -1055,6 +1062,21 @@ pub struct BlueprintSledConfig { pub state: SledState, pub subnet: Ipv6Subnet, + /// Each sled is assigned an IPv6 /64 `subnet` (above). Currently, the + /// lowest /112 subnet within the sled subnet is reserved for control plane + /// services, and of that the lowest 32 IPs are reserved for special zones + /// (global zone, switch zone) and rack setup. + /// `last_allocated_ip_subnet_offset` therefore starts at 32 for new sleds, + /// and the planner chooses IP addresses for new zones by incrementing it + /// and taking that offset into `subnet`. + /// + /// Planning will fail if `last_allocated_ip_subnet_offset` reaches + /// `u16::MAX`. This is very unlikely given current rates of updates and IP + /// assignments, but is well within the realm of "possible". Giving + /// Reconfigurator a larger chunk of IPs is tracked by + /// . + pub last_allocated_ip_subnet_offset: u16, + /// Generation number used when this type is converted into an /// `OmicronSledConfig` for use by sled-agent. /// @@ -1074,6 +1096,13 @@ pub struct BlueprintSledConfig { } impl BlueprintSledConfig { + pub fn last_allocated_ip(&self) -> Ipv6Addr { + self.subnet + .net() + .prefix() + .saturating_add(u128::from(self.last_allocated_ip_subnet_offset)) + } + /// Converts self into [`OmicronSledConfig`]. /// /// This function is effectively a `From` implementation, but diff --git a/nexus/types/src/deployment/blueprint_display.rs b/nexus/types/src/deployment/blueprint_display.rs index bfa51da9fcb..a202a171a9c 100644 --- a/nexus/types/src/deployment/blueprint_display.rs +++ b/nexus/types/src/deployment/blueprint_display.rs @@ -56,6 +56,7 @@ pub mod constants { pub const STATE: &str = "state"; pub const CONFIG_GENERATION: &str = "config generation"; pub const SUBNET: &str = "subnet"; + pub const LAST_ALLOCATED_IP: &str = "last allocated IP"; } use constants::*; use std::fmt::Display; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index d0ef5cb7cf7..d999c070013 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4815,6 +4815,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_metadata ( -- the sled's /64 subnet on the underlay address subnet INET NOT NULL, + -- the last allocated IP within `subnet` used by the blueprint + last_allocated_ip_subnet_offset INT4 NOT NULL, + PRIMARY KEY (blueprint_id, sled_id) ); diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index f511f1de056..4914b557a37 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -914,6 +914,7 @@ impl Plan { BlueprintSledConfig { state: SledState::Active, subnet: sled_description.subnet, + last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, sled_agent_generation: sled_agent_config_generation, disks: sled_config.disks.clone(), datasets, From ce625ff511735174098c0463ba4919f104869ac7 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 17 Dec 2025 16:10:46 -0500 Subject: [PATCH 2/8] use last_allocated_ip_subnet_offset when constructing SledUnderlayIpAllocator --- .../src/blueprint_editor/sled_editor.rs | 23 +++------- .../sled_editor/underlay_ip_allocator.rs | 46 +++++++++++-------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs index 0ee71c1da21..a9e5b06f4b0 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs @@ -36,6 +36,7 @@ use nexus_types::deployment::blueprint_zone_type; use nexus_types::external_api::views::SledState; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; +use omicron_common::address::SLED_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::disk::DatasetKind; use omicron_common::disk::M2Slot; @@ -47,7 +48,6 @@ use omicron_uuid_kinds::ZpoolUuid; use scalar::ScalarEditor; use sled_agent_types::inventory::MupdateOverrideBootInventory; use sled_agent_types::inventory::ZoneKind; -use std::iter; use std::mem; use std::net::Ipv6Addr; use underlay_ip_allocator::SledUnderlayIpAllocator; @@ -504,17 +504,10 @@ impl ActiveSledEditor { let zones = ZonesEditor::new(config.sled_agent_generation, config.zones); - // We never reuse underlay IPs within a sled, regardless of zone - // dispositions. If a zone has been fully removed from the blueprint - // some time after expungement, we may reuse its IP; reconfigurator must - // know that's safe prior to pruning the expunged zone. - let zone_ips = - zones.zones(BlueprintZoneDisposition::any).map(|z| z.underlay_ip()); - Ok(Self { underlay_ip_allocator: SledUnderlayIpAllocator::new( config.subnet, - zone_ips, + config.last_allocated_ip_subnet_offset, ), incoming_sled_agent_generation: config.sled_agent_generation, zones, @@ -529,15 +522,11 @@ impl ActiveSledEditor { } pub fn new_empty(subnet: Ipv6Subnet) -> Self { - // Creating the underlay IP allocator can only fail if we have a zone - // with an IP outside the sled subnet, but we don't have any zones at - // all, so this can't fail. Match explicitly to guard against this error - // turning into an enum and getting new variants we'd need to check. - let underlay_ip_allocator = - SledUnderlayIpAllocator::new(subnet, iter::empty()); - Self { - underlay_ip_allocator, + underlay_ip_allocator: SledUnderlayIpAllocator::new( + subnet, + SLED_RESERVED_ADDRESSES, + ), incoming_sled_agent_generation: Generation::new(), zones: ZonesEditor::empty(), disks: DisksEditor::empty(), diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs index 0b8bfced44a..943cec5cbd6 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs @@ -31,13 +31,12 @@ pub(crate) struct SledUnderlayIpAllocator { impl SledUnderlayIpAllocator { /// Create a new allocator for the given sled subnet that reserves all the - /// specified IPs. - /// - /// Fails if any of the specified IPs are not part of the sled subnet. - pub fn new(sled_subnet: Ipv6Subnet, in_use_ips: I) -> Self - where - I: Iterator, - { + /// IPs from the "reserved for control plane usage" block up through + /// `last_allocated_ip_subnet_offset`. + pub fn new( + sled_subnet: Ipv6Subnet, + last_allocated_ip_subnet_offset: u16, + ) -> Self { let sled_subnet_addr = sled_subnet.net().prefix(); let minimum = sled_subnet_addr .saturating_add(u128::from(SLED_RESERVED_ADDRESSES)); @@ -57,10 +56,20 @@ impl SledUnderlayIpAllocator { assert!(sled_subnet.net().contains(minimum)); assert!(sled_subnet.net().contains(maximum)); - let mut slf = Self { subnet: sled_subnet, last: minimum, maximum }; - for ip in in_use_ips { - slf.mark_as_allocated(ip); - } + // We ought to confirm that `last_allocated_ip_subnet_offset` isn't + // beyond `CP_SERVICES_RESERVED_ADDRESSES`, but that's guaranteed by the + // value of `CP_SERVICES_RESERVED_ADDRESSES`. Statically assert that + // this doesn't change; if it does, we should check that + // `last_allocated_ip_subnet_offset <= CP_SERVICES_RESERVED_ADDRESSES`. + static_assertions::const_assert_eq!( + CP_SERVICES_RESERVED_ADDRESSES, + u16::MAX + ); + + let last_allocated_ip = sled_subnet_addr + .saturating_add(u128::from(last_allocated_ip_subnet_offset)); + let last = Ipv6Addr::max(last_allocated_ip, minimum); + let slf = Self { subnet: sled_subnet, last, maximum }; assert!(minimum <= slf.last); assert!(slf.last < slf.maximum); @@ -94,10 +103,9 @@ impl SledUnderlayIpAllocator { offset >= SLED_RESERVED_ADDRESSES, "offset unexpectedly inside reserved range: {offset}" ); - assert!( - offset <= CP_SERVICES_RESERVED_ADDRESSES, - "offset unexpectedly above reserved range: {offset}" - ); + // We should also assert `offset <= CP_SERVICES_RESERVED_ADDRESSES`, but + // the latter is set to u16::MAX, so clippy (correctly) complains that + // we're asserting something that can never be false. offset } @@ -152,8 +160,7 @@ mod test { ]; let reserved_ips = reserved.iter().copied().collect::>(); - let mut allocator = - SledUnderlayIpAllocator::new(sled_subnet, reserved.iter().copied()); + let mut allocator = SledUnderlayIpAllocator::new(sled_subnet, 0xd7); let mut allocated = Vec::new(); for _ in 0..16 { @@ -167,9 +174,8 @@ mod test { assert_eq!( allocated, [ - // Because fd00::d7 was reserved, everything up to it is also - // skipped. It doesn't have to work that way, but it currently - // does. + // Because fd00::d7 is the highest we've previously allocated, + // all new allocations start just after it. "fd00::d8".parse::().unwrap(), "fd00::d9".parse().unwrap(), "fd00::da".parse().unwrap(), From 5b9dd0599c7a297edf269b18564659878b79b7ae Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 10:41:20 -0500 Subject: [PATCH 3/8] schema migration --- nexus/db-model/src/schema_versions.rs | 3 +- nexus/tests/integration_tests/schema.rs | 260 ++++++++++++++++++ .../crdb/blueprint-sled-last-used-ip/up01.sql | 2 + .../crdb/blueprint-sled-last-used-ip/up02.sql | 59 ++++ .../crdb/blueprint-sled-last-used-ip/up03.sql | 2 + schema/crdb/dbinit.sql | 2 +- 6 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 schema/crdb/blueprint-sled-last-used-ip/up01.sql create mode 100644 schema/crdb/blueprint-sled-last-used-ip/up02.sql create mode 100644 schema/crdb/blueprint-sled-last-used-ip/up03.sql diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index da17bf04dfc..ab1abd0fe68 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(214, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(215, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(215, "blueprint-sled-last-used-ip"), KnownVersion::new(214, "separate-transit-ips-by-version"), KnownVersion::new(213, "fm-cases"), KnownVersion::new(212, "local-storage-disk-type"), diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 8a50c863d1f..aea808ec135 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -3781,6 +3781,260 @@ mod migration_211 { } } +mod migration_215 { + use super::*; + use pretty_assertions::assert_eq; + use std::collections::BTreeMap; + + // randomly-generated IDs + const SLED_ID_1: &str = "ee1f2a5e-6b82-4487-8e78-779cf2b0a860"; + const SLED_ID_2: &str = "a653a2d2-91f7-4604-adda-9bcc525db254"; + const SLED_ID_3: &str = "8317bea6-bebc-4e58-ae09-8f1cef72ff4d"; + + const BP_ID_1: &str = "64ec2dba-922c-43b3-a576-699c3564d729"; + const BP_ID_2: &str = "dd58e3d3-2760-48e9-8669-6ff0f076ba84"; + const BP_ID_3: &str = "046b9d8f-f152-4370-86b8-e48445670aff"; + + const SLED_SUBNET_1: &str = "fd00:1122:3344:0101"; + const SLED_SUBNET_2: &str = "fd00:1122:3344:0102"; + const SLED_SUBNET_3: &str = "fd00:1122:3344:0103"; + + async fn before_impl(ctx: &MigrationContext<'_>) { + // Blueprint 1: 2 sleds with 3 zones each, IPs with final hextet both + // above and below ::20 (SLED_RESERVED_ADDRESSES). + // + // Blueprint 2: all 3 sleds, but the third sled has no zones + // + // Blueprint 3: all 3 sleds with zones, but the third has no zones with + // IPs above ::20. + ctx.client + .batch_execute(&format!( + " + -- Remove detritus from earlier migrations. + DELETE FROM omicron.public.bp_sled_metadata WHERE 1=1; + DELETE FROM omicron.public.bp_omicron_zone WHERE 1=1; + + -- Blueprint 1 + INSERT INTO omicron.public.bp_sled_metadata ( + blueprint_id, sled_id, sled_state, sled_agent_generation, + remove_mupdate_override, host_phase_2_desired_slot_a, + host_phase_2_desired_slot_b, subnet + ) + VALUES + ('{BP_ID_1}', '{SLED_ID_1}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_1}::/64'), + ('{BP_ID_1}', '{SLED_ID_2}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_2}::/64'); + + -- Blueprint 1 sled 1 zones + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_1}', '{SLED_ID_1}', gen_random_uuid(), 'clickhouse', + '{SLED_SUBNET_1}::10', 8080, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_1}', '{SLED_ID_1}', gen_random_uuid(), 'oximeter', + '{SLED_SUBNET_1}::50', 8081, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_1}', '{SLED_ID_1}', gen_random_uuid(), 'crucible', + '{SLED_SUBNET_1}::100', 8082, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 1 sled 2 zones + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_1}', '{SLED_ID_2}', gen_random_uuid(), 'clickhouse', + '{SLED_SUBNET_2}::15', 8080, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_1}', '{SLED_ID_2}', gen_random_uuid(), 'oximeter', + '{SLED_SUBNET_2}::25', 8081, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_1}', '{SLED_ID_2}', gen_random_uuid(), 'crucible', + '{SLED_SUBNET_2}::200', 8082, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_1}', '{SLED_ID_2}', gen_random_uuid(), 'internal_dns', + '{SLED_SUBNET_2}::ffff', 8053, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 2 + INSERT INTO omicron.public.bp_sled_metadata ( + blueprint_id, sled_id, sled_state, sled_agent_generation, + remove_mupdate_override, host_phase_2_desired_slot_a, + host_phase_2_desired_slot_b, subnet + ) + VALUES + ('{BP_ID_2}', '{SLED_ID_1}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_1}::/64'), + ('{BP_ID_2}', '{SLED_ID_2}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_2}::/64'), + ('{BP_ID_2}', '{SLED_ID_3}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_3}::/64'); + + -- Blueprint 2 sled 1 zones + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_2}', '{SLED_ID_1}', gen_random_uuid(), 'clickhouse', + '{SLED_SUBNET_1}::150', 8080, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 2 sled 2 zones + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_2}', '{SLED_ID_2}', gen_random_uuid(), 'oximeter', + '{SLED_SUBNET_2}::250', 8081, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 2 sled 3: NO zones + + -- Blueprint 3 + INSERT INTO omicron.public.bp_sled_metadata ( + blueprint_id, sled_id, sled_state, sled_agent_generation, + remove_mupdate_override, host_phase_2_desired_slot_a, + host_phase_2_desired_slot_b, subnet + ) + VALUES + ('{BP_ID_3}', '{SLED_ID_1}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_1}::/64'), + ('{BP_ID_3}', '{SLED_ID_2}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_2}::/64'), + ('{BP_ID_3}', '{SLED_ID_3}', 'active', 1, NULL, NULL, NULL, + '{SLED_SUBNET_3}::/64'); + + -- Blueprint 3 sled 1 + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_3}', '{SLED_ID_1}', gen_random_uuid(), 'clickhouse', + '{SLED_SUBNET_1}::300', 8080, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 3 sled 2 + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_3}', '{SLED_ID_2}', gen_random_uuid(), 'oximeter', + '{SLED_SUBNET_2}::400', 8081, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + + -- Blueprint 3 sled 3 + INSERT INTO omicron.public.bp_omicron_zone ( + blueprint_id, sled_id, id, zone_type, + primary_service_ip, primary_service_port, + filesystem_pool, disposition, + disposition_expunged_ready_for_cleanup, image_source + ) + VALUES + ('{BP_ID_3}', '{SLED_ID_3}', gen_random_uuid(), 'crucible', + '{SLED_SUBNET_3}::5', 8082, gen_random_uuid(), 'in_service', + false, 'install_dataset'), + ('{BP_ID_3}', '{SLED_ID_3}', gen_random_uuid(), 'clickhouse', + '{SLED_SUBNET_3}::1f', 8080, gen_random_uuid(), 'in_service', + false, 'install_dataset'); + " + )) + .await + .expect("inserted pre-migration data"); + } + + async fn after_impl(ctx: &MigrationContext<'_>) { + let bp_id_1: Uuid = BP_ID_1.parse().unwrap(); + let bp_id_2: Uuid = BP_ID_2.parse().unwrap(); + let bp_id_3: Uuid = BP_ID_3.parse().unwrap(); + let sled_id_1: Uuid = SLED_ID_1.parse().unwrap(); + let sled_id_2: Uuid = SLED_ID_2.parse().unwrap(); + let sled_id_3: Uuid = SLED_ID_3.parse().unwrap(); + + // BP1, Sled1: max IP is ::100 + // BP1, Sled2: max IP is ::200 + // BP2, Sled1: max IP is ::150 + // BP2, Sled2: max IP is ::250 + // BP2, Sled3: no zones, default to 32 + // BP3, Sled1: max IP is ::300 + // BP3, Sled2: max IP is ::400 + // BP3, Sled3: max IP is ::1f (31); should get bumped to 32 + let expected = [ + ((bp_id_1, sled_id_1), 0x100), + ((bp_id_1, sled_id_2), 0x200), + ((bp_id_2, sled_id_1), 0x150), + ((bp_id_2, sled_id_2), 0x250), + ((bp_id_2, sled_id_3), 32), + ((bp_id_3, sled_id_1), 0x300), + ((bp_id_3, sled_id_2), 0x400), + ((bp_id_3, sled_id_3), 32), + ] + .into_iter() + .collect::>(); + + let rows = ctx + .client + .query( + " + SELECT blueprint_id, sled_id, last_allocated_ip_subnet_offset + FROM omicron.public.bp_sled_metadata + WHERE blueprint_id IN ($1, $2, $3) + ORDER BY blueprint_id, sled_id + ", + &[&bp_id_1, &bp_id_2, &bp_id_3], + ) + .await + .expect("queried post-migration data"); + + let got = rows + .into_iter() + .map(|row| { + ( + ( + row.get::<_, Uuid>("blueprint_id"), + row.get::<_, Uuid>("sled_id"), + ), + row.get::<_, i32>("last_allocated_ip_subnet_offset"), + ) + }) + .collect::>(); + + assert_eq!(expected, got); + } + + pub(super) fn before<'a>( + ctx: &'a MigrationContext<'a>, + ) -> BoxFuture<'a, ()> { + Box::pin(before_impl(ctx)) + } + + pub(super) fn after<'a>( + ctx: &'a MigrationContext<'a>, + ) -> BoxFuture<'a, ()> { + Box::pin(after_impl(ctx)) + } +} + // Lazily initializes all migration checks. The combination of Rust function // pointers and async makes defining a static table fairly painful, so we're // using lazy initialization instead. @@ -3901,6 +4155,12 @@ fn get_migration_checks() -> BTreeMap { .before(migration_211::before) .after(migration_211::after), ); + map.insert( + Version::new(215, 0, 0), + DataMigrationFns::new() + .before(migration_215::before) + .after(migration_215::after), + ); map } diff --git a/schema/crdb/blueprint-sled-last-used-ip/up01.sql b/schema/crdb/blueprint-sled-last-used-ip/up01.sql new file mode 100644 index 00000000000..e2ad0f6b24d --- /dev/null +++ b/schema/crdb/blueprint-sled-last-used-ip/up01.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.bp_sled_metadata + ADD COLUMN IF NOT EXISTS last_allocated_ip_subnet_offset INT4; diff --git a/schema/crdb/blueprint-sled-last-used-ip/up02.sql b/schema/crdb/blueprint-sled-last-used-ip/up02.sql new file mode 100644 index 00000000000..f12c1227a77 --- /dev/null +++ b/schema/crdb/blueprint-sled-last-used-ip/up02.sql @@ -0,0 +1,59 @@ +-- Working with INET values in SQL is "fun". +-- +-- In `up01.sql`, we added the new `last_allocated_ip_subnet_offset` column to +-- `bp_sled_metadata`, but left it as `NULL` for all rows. We now need to fill +-- in all rows with the correct value. The correct value is nontrivial: +-- +-- For the given blueprint and the given sled in that blueprint, look at all of +-- its `bp_omicron_zone` rows (except internal DNS; more below). Extract the +-- last hextet of each of those rows. (This has to be done via a combination of +-- inet functions and string manipulations.) Convert it to an integer. Take the +-- maximum of these integers, or 32 if there are no zones or if there are no +-- zones with a final hextet of at least 32. +-- +-- 32 is the magic value for `RSS_RESERVED_ADDRESSES` as defined in +-- `omicron-common`; Reconfigurator always starts new, empty sleds with a +-- `last_allocated_ip_subnet_offset` equal to 32. If Reconfigurator has added +-- zones to this sled in any given blueprint, it will have a zone with an IP +-- with a final hextet greater than 32, and we need to use that instead. +-- +-- Finally, all of this logic also has to ignore `internal_dns` zones, because +-- they listen on an IP that's outside the sled subnet. +-- +-- There are data migration tests that confirm this query behaves as expected. + +SET LOCAL disallow_full_table_scans = off; + +UPDATE omicron.public.bp_sled_metadata AS bpm SET + last_allocated_ip_subnet_offset = ( + -- The `COALESCE` here (plus the `WHERE final_hextet > 32` below) + -- guarantees we set every `last_allocated_ip_subnet_offset` to + -- 32-or-higher. If there are any zones that return a final hextet + -- greater than 32, we'll get that; otherwise, `MAX(final_hextet)` will + -- be `NULL` and the `COALESCE` will give us 32. + SELECT COALESCE(MAX(final_hextet), 32) FROM ( + SELECT ('x' || + lpad( + substr( + -- bitwise & the IP with a /112 hostmask, which + -- squishes this IP down to the string '::NNNN', + -- where there will be 1-4 hex-valued X characters. + host(primary_service_ip & hostmask('::/112')), + -- Final arg to `substr()`: this trims off the leading + -- `::` in the `::NNNN` we produced above. + 3), + -- Final args to `lpad()`: this ensure we always have + -- exactly 4 hex digits, left padding with 0 if needed. + 4, '0') + -- We concatenated a literal 'x' prefix onto the front of the + -- `NNNN` value we just produced, which causes `bit(16)` to + -- interpret it as a 16-bit hex value (which it is!). Then + -- convert that bitstring into an integer. + )::bit(16)::int AS final_hextet + FROM bp_omicron_zone WHERE + blueprint_id = bpm.blueprint_id + AND sled_id = bpm.sled_id + AND zone_type != 'internal_dns' + ) WHERE final_hextet > 32 + ) + WHERE last_allocated_ip_subnet_offset IS NULL; diff --git a/schema/crdb/blueprint-sled-last-used-ip/up03.sql b/schema/crdb/blueprint-sled-last-used-ip/up03.sql new file mode 100644 index 00000000000..ead801f7703 --- /dev/null +++ b/schema/crdb/blueprint-sled-last-used-ip/up03.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.bp_sled_metadata + ALTER COLUMN last_allocated_ip_subnet_offset SET NOT NULL; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index d999c070013..8efdee866e5 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -7481,7 +7481,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '214.0.0', NULL) + (TRUE, NOW(), NOW(), '215.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From 9450b588ebd1962be7daae3a121ef056d01fb4dc Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 10:43:48 -0500 Subject: [PATCH 4/8] openapi --- openapi/nexus-lockstep.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 2f9724cdb93..0ed47426342 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -2370,6 +2370,12 @@ "host_phase_2": { "$ref": "#/components/schemas/BlueprintHostPhase2DesiredSlots" }, + "last_allocated_ip_subnet_offset": { + "description": "Each sled is assigned an IPv6 /64 `subnet` (above). Currently, the lowest /112 subnet within the sled subnet is reserved for control plane services, and of that the lowest 32 IPs are reserved for special zones (global zone, switch zone) and rack setup. `last_allocated_ip_subnet_offset` therefore starts at 32 for new sleds, and the planner chooses IP addresses for new zones by incrementing it and taking that offset into `subnet`.\n\nPlanning will fail if `last_allocated_ip_subnet_offset` reaches `u16::MAX`. This is very unlikely given current rates of updates and IP assignments, but is well within the realm of \"possible\". Giving Reconfigurator a larger chunk of IPs is tracked by .", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, "remove_mupdate_override": { "nullable": true, "allOf": [ @@ -2415,6 +2421,7 @@ "datasets", "disks", "host_phase_2", + "last_allocated_ip_subnet_offset", "sled_agent_generation", "state", "subnet", From 96e19363547cd032d06ebf9e555ccc9771f6350a Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 13:07:04 -0500 Subject: [PATCH 5/8] add blippy check for IP above last allocated --- Cargo.lock | 1 + nexus/reconfigurator/blippy/Cargo.toml | 1 + nexus/reconfigurator/blippy/src/blippy.rs | 20 ++++++ nexus/reconfigurator/blippy/src/checks.rs | 75 ++++++++++++++++++++++- 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 99c2a6083f5..0040a62540e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7131,6 +7131,7 @@ dependencies = [ name = "nexus-reconfigurator-blippy" version = "0.1.0" dependencies = [ + "ipnet", "nexus-reconfigurator-planning", "nexus-types", "omicron-common", diff --git a/nexus/reconfigurator/blippy/Cargo.toml b/nexus/reconfigurator/blippy/Cargo.toml index 6a19e1ece1f..ab4647553f1 100644 --- a/nexus/reconfigurator/blippy/Cargo.toml +++ b/nexus/reconfigurator/blippy/Cargo.toml @@ -15,5 +15,6 @@ omicron-workspace-hack.workspace = true tufaceous-artifact.workspace = true [dev-dependencies] +ipnet.workspace = true nexus-reconfigurator-planning.workspace = true omicron-test-utils.workspace = true diff --git a/nexus/reconfigurator/blippy/src/blippy.rs b/nexus/reconfigurator/blippy/src/blippy.rs index 4352033f460..6a7394c3c67 100644 --- a/nexus/reconfigurator/blippy/src/blippy.rs +++ b/nexus/reconfigurator/blippy/src/blippy.rs @@ -27,6 +27,7 @@ use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeSet; use std::net::IpAddr; +use std::net::Ipv6Addr; use std::net::SocketAddrV6; use tufaceous_artifact::ArtifactHash; @@ -139,6 +140,12 @@ pub enum SledKind { zone: BlueprintZoneConfig, subnet: Ipv6Subnet, }, + /// A sled has a zone with an IP that is above the sled's overall "last + /// allocated IP" value. + UnderlayIpAboveLastAllocatedIp { + zone: BlueprintZoneConfig, + last_allocated_ip: Ipv6Addr, + }, /// Two sleds are using the same sled subnet. ConflictingSledSubnets { other_sled: SledUuid, @@ -269,6 +276,19 @@ impl fmt::Display for SledKind { subnet, ) } + SledKind::UnderlayIpAboveLastAllocatedIp { + zone, + last_allocated_ip, + } => { + write!( + f, + "{:?} zone {} underlay IP {} is above the sled's last \ + allocated IP: {last_allocated_ip}", + zone.zone_type.kind(), + zone.id, + zone.underlay_ip(), + ) + } SledKind::ConflictingSledSubnets { other_sled, subnet } => { write!( f, diff --git a/nexus/reconfigurator/blippy/src/checks.rs b/nexus/reconfigurator/blippy/src/checks.rs index cbe6abf028c..4795d21b5f1 100644 --- a/nexus/reconfigurator/blippy/src/checks.rs +++ b/nexus/reconfigurator/blippy/src/checks.rs @@ -99,7 +99,8 @@ fn check_underlay_ips(blippy: &mut Blippy<'_>) { ); } } else { - let subnet = blippy.blueprint().sleds.get(&sled_id).unwrap().subnet; + let sled_config = blippy.blueprint().sleds.get(&sled_id).unwrap(); + let subnet = sled_config.subnet; // Any given subnet should be used by at most one sled. match sled_subnets_by_subnet.entry(subnet) { @@ -133,6 +134,21 @@ fn check_underlay_ips(blippy: &mut Blippy<'_>) { }, ); } + + // Any underlay IP (other than internal DNS, which we check above) + // from this sled should be no higher than this sled's last + // allocated IP. + let last_allocated_ip = sled_config.last_allocated_ip(); + if last_allocated_ip < ip { + blippy.push_sled_note( + sled_id, + Severity::Fatal, + SledKind::UnderlayIpAboveLastAllocatedIp { + zone: zone.clone(), + last_allocated_ip, + }, + ); + } } } } @@ -744,6 +760,7 @@ mod tests { use crate::BlippyReportSortKey; use crate::blippy::Kind; use crate::blippy::Note; + use ipnet::IpAdd; use nexus_reconfigurator_planning::example::ExampleSystemBuilder; use nexus_reconfigurator_planning::example::example; use nexus_types::deployment::BlueprintArtifactVersion; @@ -851,6 +868,62 @@ mod tests { logctx.cleanup_successful(); } + #[test] + fn test_underlay_ip_above_last_allocated() { + static TEST_NAME: &str = "test_underlay_ip_above_last_allocated"; + let logctx = test_setup_log(TEST_NAME); + let (_, _, mut blueprint) = example(&logctx.log, TEST_NAME); + + // Assign an "above last" IP to a Nexus zone. + let (sled_id, sled_config) = blueprint + .sleds + .iter_mut() + .find(|(_sled_id, config)| { + config.zones.iter().any(|z| z.zone_type.is_nexus()) + }) + .expect("at least one Nexus zone"); + + // Get the current `last_allocated_ip`, then change Nexus's IP to be one + // above that. + let last_allocated_ip = sled_config.last_allocated_ip(); + let bad_ip = last_allocated_ip.saturating_add(1); + let mut nexus_config = None; + + for mut z in &mut sled_config.zones { + if let BlueprintZoneType::Nexus(nexus) = &mut z.zone_type { + nexus.internal_address.set_ip(bad_ip); + nexus_config = Some(z.into_ref().clone()); + break; + } + } + + // We know this sled has a Nexus, so the loop must have populated this. + let nexus_config = nexus_config.unwrap(); + + let expected_notes = [Note { + severity: Severity::Fatal, + kind: Kind::Sled { + sled_id: *sled_id, + kind: Box::new(SledKind::UnderlayIpAboveLastAllocatedIp { + zone: nexus_config, + last_allocated_ip, + }), + }, + }]; + + let report = Blippy::new_blueprint_only(&blueprint) + .into_report(BlippyReportSortKey::Kind); + eprintln!("{}", report.display()); + for note in expected_notes { + assert!( + report.notes().contains(¬e), + "did not find expected note {note:?}" + ); + } + + logctx.cleanup_successful(); + } + #[test] fn test_duplicate_sled_subnet() { static TEST_NAME: &str = "test_duplicate_sled_subnet"; From 8d55a7db2091e86c4bbf78810c8c499868806f01 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 15:14:45 -0500 Subject: [PATCH 6/8] Dave saves ~~Christmas~~ data migrations --- .../crdb/blueprint-sled-last-used-ip/up02.sql | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/schema/crdb/blueprint-sled-last-used-ip/up02.sql b/schema/crdb/blueprint-sled-last-used-ip/up02.sql index f12c1227a77..cd11e10f9e7 100644 --- a/schema/crdb/blueprint-sled-last-used-ip/up02.sql +++ b/schema/crdb/blueprint-sled-last-used-ip/up02.sql @@ -1,13 +1,10 @@ --- Working with INET values in SQL is "fun". --- -- In `up01.sql`, we added the new `last_allocated_ip_subnet_offset` column to -- `bp_sled_metadata`, but left it as `NULL` for all rows. We now need to fill -- in all rows with the correct value. The correct value is nontrivial: -- -- For the given blueprint and the given sled in that blueprint, look at all of -- its `bp_omicron_zone` rows (except internal DNS; more below). Extract the --- last hextet of each of those rows. (This has to be done via a combination of --- inet functions and string manipulations.) Convert it to an integer. Take the +-- last hextet of each of those rows. Convert it to an integer. Take the -- maximum of these integers, or 32 if there are no zones or if there are no -- zones with a final hextet of at least 32. -- @@ -32,24 +29,14 @@ UPDATE omicron.public.bp_sled_metadata AS bpm SET -- greater than 32, we'll get that; otherwise, `MAX(final_hextet)` will -- be `NULL` and the `COALESCE` will give us 32. SELECT COALESCE(MAX(final_hextet), 32) FROM ( - SELECT ('x' || - lpad( - substr( - -- bitwise & the IP with a /112 hostmask, which - -- squishes this IP down to the string '::NNNN', - -- where there will be 1-4 hex-valued X characters. - host(primary_service_ip & hostmask('::/112')), - -- Final arg to `substr()`: this trims off the leading - -- `::` in the `::NNNN` we produced above. - 3), - -- Final args to `lpad()`: this ensure we always have - -- exactly 4 hex digits, left padding with 0 if needed. - 4, '0') - -- We concatenated a literal 'x' prefix onto the front of the - -- `NNNN` value we just produced, which causes `bit(16)` to - -- interpret it as a 16-bit hex value (which it is!). Then - -- convert that bitstring into an integer. - )::bit(16)::int AS final_hextet + SELECT ( + -- Flatten the primary service IP down to just its final + -- hextet (16 bits). + (primary_service_ip & hostmask('::/112')) + - + -- Convert it to an integer by subtracting the base ipv6 inet + '::'::inet + ) AS final_hextet FROM bp_omicron_zone WHERE blueprint_id = bpm.blueprint_id AND sled_id = bpm.sled_id From 61502eeffb813b45a9a3b7b4afbda263a47d68a6 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 15:19:48 -0500 Subject: [PATCH 7/8] add check constraint for u16 column --- schema/crdb/blueprint-sled-last-used-ip/up01.sql | 3 ++- schema/crdb/dbinit.sql | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/schema/crdb/blueprint-sled-last-used-ip/up01.sql b/schema/crdb/blueprint-sled-last-used-ip/up01.sql index e2ad0f6b24d..300d65e77b6 100644 --- a/schema/crdb/blueprint-sled-last-used-ip/up01.sql +++ b/schema/crdb/blueprint-sled-last-used-ip/up01.sql @@ -1,2 +1,3 @@ ALTER TABLE omicron.public.bp_sled_metadata - ADD COLUMN IF NOT EXISTS last_allocated_ip_subnet_offset INT4; + ADD COLUMN IF NOT EXISTS last_allocated_ip_subnet_offset INT4 + CHECK (last_allocated_ip_subnet_offset BETWEEN 0 AND 65535); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 8efdee866e5..c0b968e81c3 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4816,7 +4816,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_metadata ( subnet INET NOT NULL, -- the last allocated IP within `subnet` used by the blueprint - last_allocated_ip_subnet_offset INT4 NOT NULL, + last_allocated_ip_subnet_offset INT4 + CHECK (last_allocated_ip_subnet_offset BETWEEN 0 AND 65535) + NOT NULL, PRIMARY KEY (blueprint_id, sled_id) ); From 27a7e0d8c7ba8bbb06389b8b67d6c74a646468b9 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Fri, 19 Dec 2025 16:10:54 -0500 Subject: [PATCH 8/8] add LastAllocatedSubnetIpOffset newtype --- common/src/address.rs | 5 -- .../db-queries/src/db/datastore/deployment.rs | 9 ++- .../reconfigurator/execution/src/database.rs | 5 +- nexus/reconfigurator/execution/src/dns.rs | 5 +- .../execution/src/omicron_sled_config.rs | 5 +- .../src/blueprint_editor/sled_editor.rs | 4 +- .../sled_editor/underlay_ip_allocator.rs | 38 +++++-------- .../background/tasks/blueprint_execution.rs | 6 +- nexus/test-utils/src/starter.rs | 5 +- nexus/types/src/deployment.rs | 55 ++++++++++++++++--- sled-agent/src/rack_setup/plan/service.rs | 10 +++- 11 files changed, 95 insertions(+), 52 deletions(-) diff --git a/common/src/address.rs b/common/src/address.rs index 0e8d18fff5b..94ff1aa9d43 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -278,11 +278,6 @@ pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF; // to assume that addresses in this subnet are available. pub const SLED_RESERVED_ADDRESSES: u16 = 32; -static_assertions::const_assert_eq!( - RSS_RESERVED_ADDRESSES, - SLED_RESERVED_ADDRESSES -); - /// Wraps an [`Ipv6Net`] with a compile-time prefix length. #[derive( Debug, diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index edd0699b122..52d39d540e7 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -83,6 +83,7 @@ use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::ClickhouseClusterConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::ExpectedVersion; +use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::OximeterReadMode; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; @@ -262,6 +263,7 @@ impl DataStore { subnet: Ipv6Network::from(sled.subnet).into(), last_allocated_ip_subnet_offset: sled .last_allocated_ip_subnet_offset + .into_u16() .into(), }) .collect::>(); @@ -799,12 +801,15 @@ impl DataStore { &InlineErrorChain::new(&*e).to_string(), ) })?; + let last_allocated_ip_subnet_offset = + LastAllocatedSubnetIpOffset::new( + *s.last_allocated_ip_subnet_offset, + ); let config = BlueprintSledConfig { state: s.sled_state.into(), subnet, sled_agent_generation: *s.sled_agent_generation, - last_allocated_ip_subnet_offset: *s - .last_allocated_ip_subnet_offset, + last_allocated_ip_subnet_offset, disks: IdOrdMap::new(), datasets: IdOrdMap::new(), zones: IdOrdMap::new(), diff --git a/nexus/reconfigurator/execution/src/database.rs b/nexus/reconfigurator/execution/src/database.rs index 4a5cfece456..10a6527e44c 100644 --- a/nexus/reconfigurator/execution/src/database.rs +++ b/nexus/reconfigurator/execution/src/database.rs @@ -79,6 +79,7 @@ mod test { use nexus_types::deployment::BlueprintZoneImageSource; use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::CockroachDbPreserveDowngrade; + use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::deployment::OximeterReadMode; use nexus_types::deployment::PendingMgsUpdates; @@ -87,7 +88,6 @@ mod test { use nexus_types::inventory::NetworkInterface; use nexus_types::inventory::NetworkInterfaceKind; use omicron_common::address::Ipv6Subnet; - use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; @@ -166,7 +166,8 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset: + LastAllocatedSubnetIpOffset::initial(), sled_agent_generation: Generation::new(), zones, disks: IdOrdMap::new(), diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index b64c36700b3..d0ba92a33aa 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -347,6 +347,7 @@ mod test { use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::ExternalIpPolicy; + use nexus_types::deployment::LastAllocatedSubnetIpOffset; pub use nexus_types::deployment::OmicronZoneExternalFloatingAddr; pub use nexus_types::deployment::OmicronZoneExternalFloatingIp; pub use nexus_types::deployment::OmicronZoneExternalSnatIp; @@ -368,7 +369,6 @@ mod test { use omicron_common::address::Ipv6Subnet; use omicron_common::address::RACK_PREFIX; use omicron_common::address::REPO_DEPOT_PORT; - use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::address::SLED_PREFIX; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; @@ -696,7 +696,8 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(*sa.sled_agent_address.ip()), - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset: + LastAllocatedSubnetIpOffset::initial(), sled_agent_generation: ledgered_sled_config.generation, disks: IdOrdMap::new(), datasets: IdOrdMap::new(), diff --git a/nexus/reconfigurator/execution/src/omicron_sled_config.rs b/nexus/reconfigurator/execution/src/omicron_sled_config.rs index 566b0752689..9f08a815c82 100644 --- a/nexus/reconfigurator/execution/src/omicron_sled_config.rs +++ b/nexus/reconfigurator/execution/src/omicron_sled_config.rs @@ -91,13 +91,13 @@ mod tests { use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; use nexus_types::deployment::BlueprintZoneType; + use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::blueprint_zone_type; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledProvisionPolicy; use nexus_types::external_api::views::SledState; use omicron_common::address::Ipv6Subnet; use omicron_common::address::REPO_DEPOT_PORT; - use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::CompressionAlgorithm; @@ -263,7 +263,8 @@ mod tests { let sled_config = BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset: + LastAllocatedSubnetIpOffset::initial(), sled_agent_generation: sim_sled_agent_config_generation.next(), disks, datasets, diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs index a9e5b06f4b0..e5c2a1f4b1d 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs @@ -31,12 +31,12 @@ use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; use nexus_types::deployment::BlueprintZoneType; +use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::blueprint_zone_type; use nexus_types::external_api::views::SledState; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; -use omicron_common::address::SLED_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::disk::DatasetKind; use omicron_common::disk::M2Slot; @@ -525,7 +525,7 @@ impl ActiveSledEditor { Self { underlay_ip_allocator: SledUnderlayIpAllocator::new( subnet, - SLED_RESERVED_ADDRESSES, + LastAllocatedSubnetIpOffset::initial(), ), incoming_sled_agent_generation: Generation::new(), zones: ZonesEditor::empty(), diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs index 943cec5cbd6..c22c264743c 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/underlay_ip_allocator.rs @@ -6,6 +6,7 @@ use ipnet::IpAdd; use ipnet::IpSub; +use nexus_types::deployment::LastAllocatedSubnetIpOffset; use omicron_common::address::CP_SERVICES_RESERVED_ADDRESSES; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; @@ -35,7 +36,7 @@ impl SledUnderlayIpAllocator { /// `last_allocated_ip_subnet_offset`. pub fn new( sled_subnet: Ipv6Subnet, - last_allocated_ip_subnet_offset: u16, + last_allocated_ip_subnet_offset: LastAllocatedSubnetIpOffset, ) -> Self { let sled_subnet_addr = sled_subnet.net().prefix(); let minimum = sled_subnet_addr @@ -56,18 +57,8 @@ impl SledUnderlayIpAllocator { assert!(sled_subnet.net().contains(minimum)); assert!(sled_subnet.net().contains(maximum)); - // We ought to confirm that `last_allocated_ip_subnet_offset` isn't - // beyond `CP_SERVICES_RESERVED_ADDRESSES`, but that's guaranteed by the - // value of `CP_SERVICES_RESERVED_ADDRESSES`. Statically assert that - // this doesn't change; if it does, we should check that - // `last_allocated_ip_subnet_offset <= CP_SERVICES_RESERVED_ADDRESSES`. - static_assertions::const_assert_eq!( - CP_SERVICES_RESERVED_ADDRESSES, - u16::MAX - ); - - let last_allocated_ip = sled_subnet_addr - .saturating_add(u128::from(last_allocated_ip_subnet_offset)); + let last_allocated_ip = + last_allocated_ip_subnet_offset.to_ip(sled_subnet); let last = Ipv6Addr::max(last_allocated_ip, minimum); let slf = Self { subnet: sled_subnet, last, maximum }; assert!(minimum <= slf.last); @@ -82,7 +73,9 @@ impl SledUnderlayIpAllocator { } /// Get the last allocated IP as an offset into the sled subnet. - pub fn last_allocated_ip_subnet_offset(&self) -> u16 { + pub fn last_allocated_ip_subnet_offset( + &self, + ) -> LastAllocatedSubnetIpOffset { let last_allocated_ip = self.last; let offset = self.last.saturating_sub(self.subnet.net().prefix()); @@ -99,15 +92,8 @@ impl SledUnderlayIpAllocator { ); } }; - assert!( - offset >= SLED_RESERVED_ADDRESSES, - "offset unexpectedly inside reserved range: {offset}" - ); - // We should also assert `offset <= CP_SERVICES_RESERVED_ADDRESSES`, but - // the latter is set to u16::MAX, so clippy (correctly) complains that - // we're asserting something that can never be false. - offset + LastAllocatedSubnetIpOffset::new(offset) } /// Mark an address as used. @@ -146,9 +132,8 @@ impl SledUnderlayIpAllocator { #[cfg(test)] mod test { - use std::collections::BTreeSet; - use super::*; + use std::collections::BTreeSet; #[test] fn test_basic() { @@ -160,7 +145,10 @@ mod test { ]; let reserved_ips = reserved.iter().copied().collect::>(); - let mut allocator = SledUnderlayIpAllocator::new(sled_subnet, 0xd7); + let mut allocator = SledUnderlayIpAllocator::new( + sled_subnet, + LastAllocatedSubnetIpOffset::new(0xd7), + ); let mut allocated = Vec::new(); for _ in 0..16 { diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index 26b5433709a..dccf9c9b8bc 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -235,6 +235,7 @@ mod test { use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_test_utils_macros::nexus_test; + use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::execution::{ EventBuffer, EventReport, ExecutionComponent, ExecutionStepId, StepOutcome, StepStatus, @@ -247,7 +248,7 @@ mod test { blueprint_zone_type, }; use nexus_types::external_api::views::SledState; - use omicron_common::address::{Ipv6Subnet, RSS_RESERVED_ADDRESSES}; + use omicron_common::address::Ipv6Subnet; use omicron_common::api::external; use omicron_common::api::external::Generation; use omicron_common::zpool_name::ZpoolName; @@ -285,7 +286,8 @@ mod test { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset: + LastAllocatedSubnetIpOffset::initial(), sled_agent_generation: Generation::new().next(), disks: IdOrdMap::new(), datasets: IdOrdMap::new(), diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index f8bd484d887..d34168c0c20 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -45,6 +45,7 @@ use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::CockroachDbPreserveDowngrade; +use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::OmicronZoneExternalFloatingAddr; use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::deployment::OmicronZoneExternalSnatIp; @@ -61,7 +62,6 @@ use omicron_common::address::Ipv6Subnet; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_PORT; -use omicron_common::address::RSS_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::api::external::Name; @@ -1390,7 +1390,8 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { BlueprintSledConfig { state: SledState::Active, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset: + LastAllocatedSubnetIpOffset::initial(), sled_agent_generation, disks, datasets, diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index be0fd3784e4..66f057ae32c 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -31,6 +31,7 @@ use iddqd::id_upcast; use ipnet::IpAdd; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; +use omicron_common::address::SLED_RESERVED_ADDRESSES; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::api::external::TufArtifactMeta; @@ -1071,11 +1072,11 @@ pub struct BlueprintSledConfig { /// and taking that offset into `subnet`. /// /// Planning will fail if `last_allocated_ip_subnet_offset` reaches - /// `u16::MAX`. This is very unlikely given current rates of updates and IP - /// assignments, but is well within the realm of "possible". Giving + /// its maximal value. This is very unlikely given current rates of updates + /// and IP assignments, but is well within the realm of "possible". Giving /// Reconfigurator a larger chunk of IPs is tracked by /// . - pub last_allocated_ip_subnet_offset: u16, + pub last_allocated_ip_subnet_offset: LastAllocatedSubnetIpOffset, /// Generation number used when this type is converted into an /// `OmicronSledConfig` for use by sled-agent. @@ -1097,10 +1098,7 @@ pub struct BlueprintSledConfig { impl BlueprintSledConfig { pub fn last_allocated_ip(&self) -> Ipv6Addr { - self.subnet - .net() - .prefix() - .saturating_add(u128::from(self.last_allocated_ip_subnet_offset)) + self.last_allocated_ip_subnet_offset.to_ip(self.subnet) } /// Converts self into [`OmicronSledConfig`]. @@ -1176,6 +1174,49 @@ impl BlueprintSledConfig { } } +/// Offset stored within [`BlueprintSledConfig`] indicating the last IP +/// Reconfigurator has allocated within that sled's subnet. +/// +/// The inner value has a minimum of [`SLED_RESERVED_ADDRESSES`]; we treat all +/// of those as "already allocated". +#[derive( + Debug, Clone, Copy, Eq, PartialEq, JsonSchema, Serialize, Diffable, +)] +#[serde(transparent)] +pub struct LastAllocatedSubnetIpOffset(u16); + +impl LastAllocatedSubnetIpOffset { + pub fn initial() -> Self { + Self(SLED_RESERVED_ADDRESSES) + } + + /// Construct a new offset, enforcing our lower bound of + /// [`SLED_RESERVED_ADDRESSES`]. + pub fn new(offset: u16) -> Self { + let offset = u16::max(offset, SLED_RESERVED_ADDRESSES); + Self(offset) + } + + /// Convert this offset into the `offset`'th IP in `subnet`. + pub fn to_ip(self, subnet: Ipv6Subnet) -> Ipv6Addr { + subnet.net().prefix().saturating_add(u128::from(self.0)) + } + + pub fn into_u16(self) -> u16 { + self.0 + } +} + +impl<'de> Deserialize<'de> for LastAllocatedSubnetIpOffset { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let inner = u16::deserialize(deserializer)?; + Ok(LastAllocatedSubnetIpOffset::new(inner)) + } +} + trait ZoneSortKey { fn kind(&self) -> ZoneKind; fn id(&self) -> OmicronZoneUuid; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 4914b557a37..b8d4f24dbe2 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -15,6 +15,7 @@ use internal_dns_types::config::{ DnsConfigBuilder, DnsConfigParams, Host, Zone, }; use internal_dns_types::names::ServiceName; +use nexus_types::deployment::LastAllocatedSubnetIpOffset; use nexus_types::deployment::{ Blueprint, BlueprintDatasetConfig, BlueprintDatasetDisposition, BlueprintHostPhase2DesiredSlots, BlueprintPhysicalDiskConfig, @@ -909,12 +910,19 @@ impl Plan { })?; } + // We could more carefully track which IPs we actually allocated to + // this sled, but in practice Reconfigurator treats the entire + // RSS_RESERVED_ADDRESSES range as reserved anyway, so it's fine for + // us to just start there. + let last_allocated_ip_subnet_offset = + LastAllocatedSubnetIpOffset::new(RSS_RESERVED_ADDRESSES); + blueprint_sleds.insert( sled_description.sled_id, BlueprintSledConfig { state: SledState::Active, subnet: sled_description.subnet, - last_allocated_ip_subnet_offset: RSS_RESERVED_ADDRESSES, + last_allocated_ip_subnet_offset, sled_agent_generation: sled_agent_config_generation, disks: sled_config.disks.clone(), datasets,