AWS peer discovery: add multi-hostname path support

BenAtAmazon · lukebakken · commit c0c79bfd6324 · 2025-10-22T08:18:42.000-07:00
Adds support for multiple hostname paths in the AWS peer discovery plugin to enable zero-downtime rolling upgrades during hostname migration scenarios.

The implementation allows RabbitMQ nodes to discover peers using multiple hostname paths, ensuring cluster formation succeeds even when nodes are configured with different hostname paths during rolling upgrades.

Example usage:

cluster_formation.aws.hostname_path.1 = privateDnsName
cluster_formation.aws.hostname_path.2 = privateIpAddress

Move tests that need per-test init and end to dedicated file.
diff --git a/deps/rabbitmq_peer_discovery_aws/priv/schema/rabbitmq_peer_discovery_aws.schema b/deps/rabbitmq_peer_discovery_aws/priv/schema/rabbitmq_peer_discovery_aws.schema
@@ -108,3 +108,33 @@ fun(Conf) ->
         Value     -> rabbit_peer_discovery_util:as_list(Value)
     end
 end}.
+
+%% hostname_paths (multiple paths support)
+
+{mapping, "cluster_formation.aws.hostname_path.$number", "rabbit.cluster_formation.peer_discovery_aws.aws_hostname_paths", [
+    {datatype, string}
+]}.
+
+{translation, "rabbit.cluster_formation.peer_discovery_aws.aws_hostname_paths",
+fun(Conf) ->
+    Settings = cuttlefish_variable:filter_by_prefix("cluster_formation.aws.hostname_path", Conf),
+    %% Helper function for number validation
+    IsNumberString = fun(Str) ->
+        case string:to_integer(Str) of
+            {_, ""} -> true;
+            _ -> false
+        end
+    end,
+    %% Extract and sort numbered paths (hostname_path.1, hostname_path.2, etc.)
+    NumberedPaths = [{list_to_integer(N), V} || 
+                     {["cluster_formation", "aws", "hostname_path", N], V} <- Settings,
+                     IsNumberString(N)],
+    case NumberedPaths of
+        [] -> cuttlefish:unset();
+        _ ->
+            SortedPaths = lists:sort(NumberedPaths),
+            [rabbit_peer_discovery_util:as_list(V) || {_, V} <- SortedPaths]
+    end
+end}.
+
+
diff --git a/deps/rabbitmq_peer_discovery_aws/src/rabbit_peer_discovery_aws.erl b/deps/rabbitmq_peer_discovery_aws/src/rabbit_peer_discovery_aws.erl
@@ -62,6 +62,11 @@
                                                    env_variable  = "AWS_HOSTNAME_PATH",
                                                    default_value = ["privateDnsName"]
                                                   },
+          aws_hostname_paths                => #peer_discovery_config_entry_meta{
+                                                   type          = list,
+                                                   env_variable  = "AWS_HOSTNAME_PATHS",
+                                                   default_value = []
+                                                  },
           aws_use_private_ip                 => #peer_discovery_config_entry_meta{
                                                    type          = atom,
                                                    env_variable  = "AWS_USE_PRIVATE_IP",
@@ -318,10 +323,11 @@ get_hostname_name_from_reservation_set([], Accum) -> Accum;
 get_hostname_name_from_reservation_set([{"item", RI}|T], Accum) ->
     InstancesSet = proplists:get_value("instancesSet", RI),
     Items = [Item || {"item", Item} <- InstancesSet],
-    HostnamePath = get_hostname_path(),
-    Hostnames = [get_hostname(HostnamePath, Item) || Item <- Items],
-    Hostnames2 = [Name || Name <- Hostnames, Name =/= ""],
-    get_hostname_name_from_reservation_set(T, Accum ++ Hostnames2).
+    HostnamePaths = get_hostname_paths(),
+    ?LOG_DEBUG("AWS peer discovery: processing reservation with ~p instances using hostname paths: ~tp", 
+              [length(Items), HostnamePaths]),
+    UniqueHostnames = extract_unique_hostnames(HostnamePaths, Items),
+    get_hostname_name_from_reservation_set(T, Accum ++ UniqueHostnames).
 
 get_hostname_names(Path) ->
     case rabbitmq_aws:api_get_request("ec2", Path) of
@@ -347,31 +353,69 @@ get_hostname_by_tags(Tags) ->
             Names
     end.
 
--spec get_hostname_path() -> path().
-get_hostname_path() ->
-    UsePrivateIP = get_config_key(aws_use_private_ip, ?CONFIG_MODULE:config_map(?BACKEND_CONFIG_KEY)),
-    HostnamePath = get_config_key(aws_hostname_path, ?CONFIG_MODULE:config_map(?BACKEND_CONFIG_KEY)),
-    FinalPath = case HostnamePath of
-        ["privateDnsName"] when UsePrivateIP -> ["privateIpAddress"];
-        P -> P
+-spec get_hostname_paths() -> [path()].
+get_hostname_paths() ->
+    M = ?CONFIG_MODULE:config_map(?BACKEND_CONFIG_KEY),
+    UsePrivateIP = get_config_key(aws_use_private_ip, M),
+    RawPaths = case get_config_key(aws_hostname_paths, M) of
+        Paths when is_list(Paths), Paths =/= [] ->
+            ?LOG_DEBUG("AWS peer discovery using multiple hostname paths"),
+            Paths;
+        _ ->
+            %% Use single path configuration (including when Paths is [])
+            SinglePath = get_single_hostname_path_raw(M),
+            ?LOG_DEBUG("AWS peer discovery using single hostname path"),
+            [SinglePath]
     end,
-    ?LOG_DEBUG("AWS peer discovery using hostname path: ~tp", [FinalPath]),
-    FinalPath.
+    %% Apply use_private_ip override to all paths consistently
+    FinalPaths = apply_private_ip_override(RawPaths, UsePrivateIP),
+    ?LOG_DEBUG("AWS peer discovery final hostname paths: ~tp", [FinalPaths]),
+    FinalPaths.
+
+-spec get_single_hostname_path_raw(map()) -> path().
+get_single_hostname_path_raw(ConfigMap) ->
+    case get_config_key(aws_hostname_path, ConfigMap) of
+        undefined ->
+            ["privateDnsName"];
+        P ->
+            P
+    end.
+
+-spec apply_private_ip_override([path()], boolean()) -> [path()].
+apply_private_ip_override(Paths, UsePrivateIP) ->
+    apply_private_ip_override(Paths, UsePrivateIP, []).
+apply_private_ip_override([], _, Acc) ->
+    lists:reverse(Acc);
+apply_private_ip_override([["privateDnsName"] | Paths], true, Acc0) ->
+    Acc1 = [["privateIpAddress"] | Acc0],
+    apply_private_ip_override(Paths, true, Acc1);
+apply_private_ip_override([Path | Paths], UsePrivateIP, Acc0) ->
+    Acc1 = [Path | Acc0],
+    apply_private_ip_override(Paths, UsePrivateIP, Acc1).
 
 -spec get_hostname(path(), props()) -> string().
+get_hostname([], _Props) -> 
+    ?LOG_DEBUG("AWS peer discovery: empty hostname path provided"),
+    "";  %% Handle empty paths gracefully
 get_hostname(Path, Props) ->
     List = lists:foldl(fun get_value/2, Props, Path),
     case io_lib:latin1_char_list(List) of
-        true -> List;
-        _ -> ""
+        true -> 
+            ?LOG_DEBUG("AWS peer discovery: extracted hostname '~ts' from path ~tp", [List, Path]),
+            List;
+        _ -> 
+            ?LOG_DEBUG("AWS peer discovery: invalid hostname format from path ~tp, result: ~tp", [Path, List]),
+            ""
     end.
 
 -spec get_value(string()|integer(), props()) -> props().
-get_value(_, []) ->
-    [];
-get_value(Key, Props) when is_integer(Key) ->
-    {"item", Props2} = lists:nth(Key, Props),
-    Props2;
+get_value(Key, Props) when is_integer(Key), is_list(Props), length(Props) >= Key, Key > 0 ->
+    case lists:nth(Key, Props) of
+        {"item", Props2} -> Props2;
+        _ -> []  % Malformed data
+    end;
+get_value(Key, _Props) when is_integer(Key) ->
+    [];  % Out of bounds or empty list
 get_value(Key, Props) ->
     Value = proplists:get_value(Key, Props),
     sort_ec2_hostname_path_set_members(Key, Value).
@@ -413,3 +457,18 @@ get_tags() ->
             maps:from_list(Value);
         _ -> Tags
     end.
+
+%% Helper functions for multiple hostname paths support
+
+-spec extract_unique_hostnames([path()], [props()]) -> [string()].
+extract_unique_hostnames(Paths, Items) ->
+    ?LOG_DEBUG("AWS peer discovery: extracting hostnames using ~p paths for ~p items", 
+              [length(Paths), length(Items)]),
+    %% Extract all hostnames from all paths for all items
+    AllHostnames = [get_hostname(Path, Item) || Path <- Paths, Item <- Items],
+    %% Filter out empty hostnames and remove duplicates
+    ValidHostnames = [Name || Name <- AllHostnames, Name =/= ""],
+    UniqueHostnames = lists:uniq(ValidHostnames),
+    ?LOG_DEBUG("AWS peer discovery: extracted ~p total hostnames, ~p valid, ~p unique: ~tp", 
+              [length(AllHostnames), length(ValidHostnames), length(UniqueHostnames), UniqueHostnames]),
+    UniqueHostnames.
diff --git a/deps/rabbitmq_peer_discovery_aws/test/config_schema_SUITE_data/rabbitmq_peer_discovery_aws.snippets b/deps/rabbitmq_peer_discovery_aws/test/config_schema_SUITE_data/rabbitmq_peer_discovery_aws.snippets
@@ -103,5 +103,35 @@
                 ]}
             ]}
         ], [rabbitmq_peer_discovery_aws]
+    },
+    {aws_hostname_paths_multiple,
+        "cluster_formation.aws.hostname_path.1 = networkInterfaceSet,2,privateIpAddressesSet,1,privateDnsName
+         cluster_formation.aws.hostname_path.2 = privateDnsName
+         cluster_formation.aws.hostname_path.3 = privateIpAddress",
+        [
+            {rabbit, [
+                {cluster_formation, [
+                    {peer_discovery_aws, [
+                        {aws_hostname_paths, [
+                            ["networkInterfaceSet", 2, "privateIpAddressesSet", 1, "privateDnsName"],
+                            ["privateDnsName"],
+                            ["privateIpAddress"]
+                        ]}
+                    ]}
+                ]}
+            ]}
+        ], [rabbitmq_peer_discovery_aws]
+    },
+    {aws_hostname_paths_single_numbered,
+        "cluster_formation.aws.hostname_path.1 = privateDnsName",
+        [
+            {rabbit, [
+                {cluster_formation, [
+                    {peer_discovery_aws, [
+                        {aws_hostname_paths, [["privateDnsName"]]}
+                    ]}
+                ]}
+            ]}
+        ], [rabbitmq_peer_discovery_aws]
     }
 ].
diff --git a/deps/rabbitmq_peer_discovery_aws/test/unit_SUITE.erl b/deps/rabbitmq_peer_discovery_aws/test/unit_SUITE.erl
@@ -8,10 +8,10 @@
 -module(unit_SUITE).
 
 -compile(export_all).
+
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
-
 all() ->
     [
      {group, unit},
@@ -22,7 +22,6 @@ groups() ->
     [
      {unit, [], [
                  maybe_add_tag_filters,
-                 get_hostname_name_from_reservation_set,
                  registration_support,
                  network_interface_sorting,
                  private_ip_address_sorting
@@ -46,48 +45,7 @@ maybe_add_tag_filters(_Config) ->
                      {"Filter.1.Name", "tag:region"},
                      {"Filter.1.Value.1", "us-west-2"}]),
     Result = lists:sort(rabbit_peer_discovery_aws:maybe_add_tag_filters(Tags, [], 1)),
-    ?assertEqual(Expectation, Result).
-
-get_hostname_name_from_reservation_set(_Config) ->
-    ok = eunit:test({
-      foreach,
-      fun on_start/0,
-      fun on_finish/1,
-      [{"from private DNS",
-        fun() ->
-                Expectation = ["ip-10-0-16-29.eu-west-1.compute.internal",
-                               "ip-10-0-16-31.eu-west-1.compute.internal"],
-                ?assertEqual(Expectation,
-                             rabbit_peer_discovery_aws:get_hostname_name_from_reservation_set(
-                               reservation_set(), []))
-        end},
-       {"from arbitrary path",
-        fun() ->
-                os:putenv("AWS_HOSTNAME_PATH", "networkInterfaceSet,1,association,publicDnsName"),
-                Expectation = ["ec2-203-0-113-11.eu-west-1.compute.amazonaws.com",
-                               "ec2-203-0-113-21.eu-west-1.compute.amazonaws.com"],
-                ?assertEqual(Expectation,
-                             rabbit_peer_discovery_aws:get_hostname_name_from_reservation_set(
-                               reservation_set(), []))
-        end},
-       {"from private IP",
-        fun() ->
-                os:putenv("AWS_USE_PRIVATE_IP", "true"),
-                Expectation = ["10.0.16.29", "10.0.16.31"],
-                ?assertEqual(Expectation,
-                             rabbit_peer_discovery_aws:get_hostname_name_from_reservation_set(
-                               reservation_set(), []))
-        end},
-       {"from private IP DNS in network interface",
-        fun() ->
-                os:putenv("AWS_HOSTNAME_PATH", "networkInterfaceSet,2,privateIpAddressesSet,1,privateDnsName"),
-                Expectation = ["ip-10-0-15-100.eu-west-1.compute.internal",
-                               "ip-10-0-16-31.eu-west-1.compute.internal"],
-                ?assertEqual(Expectation,
-                             rabbit_peer_discovery_aws:get_hostname_name_from_reservation_set(
-                               reservation_set(), []))
-        end}]
-    }).
+    ?assertMatch(Expectation, Result).
 
 registration_support(_Config) ->
     ?assertEqual(false, rabbit_peer_discovery_aws:supports_registration()).
@@ -189,103 +147,3 @@ lock_multiple_nodes(_Config) ->
 lock_local_node_not_discovered(_Config) ->
   Expectation = {error, "Local node " ++ atom_to_list(node()) ++ " is not part of discovered nodes [me@host]"},
   ?assertEqual(Expectation, rabbit_peer_discovery_aws:lock([me@host])).
-
-%%%
-%%% Implementation
-%%%
-
-on_start() ->
-    reset().
-
-on_finish(_Config) ->
-    reset().
-
-reset() ->
-    application:unset_env(rabbit, cluster_formation),
-    os:unsetenv("AWS_HOSTNAME_PATH"),
-    os:unsetenv("AWS_USE_PRIVATE_IP").
-
-reservation_set() ->
-    [{"item", [{"reservationId","r-006cfdbf8d04c5f01"},
-               {"ownerId","248536293561"},
-               {"groupSet",[]},
-               {"instancesSet",
-                [{"item",
-                  [{"instanceId","i-0c6d048641f09cad2"},
-                   {"imageId","ami-ef4c7989"},
-                   {"instanceState",
-                    [{"code","16"},{"name","running"}]},
-                   {"privateDnsName",
-                    "ip-10-0-16-29.eu-west-1.compute.internal"},
-                   {"dnsName",[]},
-                   {"instanceType","c4.large"},
-                   {"launchTime","2017-04-07T12:05:10"},
-                   {"subnetId","subnet-61ff660"},
-                   {"vpcId","vpc-4fe1562b"},
-                   {"networkInterfaceSet", [
-                    {"item",
-                    [{"attachment", [{"deviceIndex", "1"}]},
-                     {"association",
-                     [{"publicIp","203.0.113.12"},
-                      {"publicDnsName",
-                       "ec2-203-0-113-12.eu-west-1.compute.amazonaws.com"},
-                      {"ipOwnerId","amazon"}]},
-                     {"privateIpAddressesSet", [
-                        {"item", [
-                            {"privateIpAddress", "10.0.15.101"},
-                            {"privateDnsName", "ip-10-0-15-101.eu-west-1.compute.internal"},
-                            {"primary", "false"}
-                        ]},
-                        {"item", [
-                            {"privateIpAddress", "10.0.15.100"},
-                            {"privateDnsName", "ip-10-0-15-100.eu-west-1.compute.internal"},
-                            {"primary", "true"}
-                        ]}
-                     ]}]},
-                    {"item",
-                    [{"attachment", [{"deviceIndex", "0"}]},
-                     {"association",
-                     [{"publicIp","203.0.113.11"},
-                      {"publicDnsName",
-                       "ec2-203-0-113-11.eu-west-1.compute.amazonaws.com"},
-                      {"ipOwnerId","amazon"}]}]}]},
-                   {"privateIpAddress","10.0.16.29"}]}]}]},
-     {"item", [{"reservationId","r-006cfdbf8d04c5f01"},
-               {"ownerId","248536293561"},
-               {"groupSet",[]},
-               {"instancesSet",
-                [{"item",
-                  [{"instanceId","i-1c6d048641f09cad2"},
-                   {"imageId","ami-af4c7989"},
-                   {"instanceState",
-                    [{"code","16"},{"name","running"}]},
-                   {"privateDnsName",
-                    "ip-10-0-16-31.eu-west-1.compute.internal"},
-                   {"dnsName",[]},
-                   {"instanceType","c4.large"},
-                   {"launchTime","2017-04-07T12:05:10"},
-                   {"subnetId","subnet-61ff660"},
-                   {"vpcId","vpc-4fe1562b"},
-                   {"networkInterfaceSet", [
-                    {"item",
-                    [{"attachment", [{"deviceIndex", "0"}]},
-                     {"association",
-                     [{"publicIp","203.0.113.21"},
-                      {"publicDnsName",
-                       "ec2-203-0-113-21.eu-west-1.compute.amazonaws.com"},
-                      {"ipOwnerId","amazon"}]}]},
-                    {"item",
-                    [{"attachment", [{"deviceIndex", "1"}]},
-                     {"association",
-                     [{"publicIp","203.0.113.22"},
-                      {"publicDnsName",
-                       "ec2-203-0-113-22.eu-west-1.compute.amazonaws.com"},
-                      {"ipOwnerId","amazon"}]},
-                     {"privateIpAddressesSet", [
-                        {"item", [
-                            {"privateIpAddress", "10.0.16.31"},
-                            {"privateDnsName", "ip-10-0-16-31.eu-west-1.compute.internal"},
-                            {"primary", "true"}
-                        ]}
-                     ]}]}]},
-                   {"privateIpAddress","10.0.16.31"}]}]}]}].
diff --git a/deps/rabbitmq_peer_discovery_aws/test/unit_reservation_set_SUITE.erl b/deps/rabbitmq_peer_discovery_aws/test/unit_reservation_set_SUITE.erl
