From 4369431cb81480c8214b22c2220d016bcac8ada6 Mon Sep 17 00:00:00 2001 From: drappier-charles Date: Sun, 2 Nov 2025 20:50:51 +0100 Subject: [PATCH 1/4] Enhance EC2 provider to support custom IMDS endpoint - Added a new environment variable `AWS_EC2_METADATA_SERVICE_ENDPOINT` to allow users to specify a custom IMDS endpoint. - Updated the EC2 provider creation logic to check for this environment variable and adjust the upstream connection accordingly. - Implemented a new test case to verify the functionality of the custom endpoint. These changes improve flexibility for users needing to connect to non-default IMDS endpoints. --- include/fluent-bit/aws/flb_aws_imds.h | 3 ++ src/aws/flb_aws_credentials_ec2.c | 41 +++++++++++++++++++++++++-- tests/internal/aws_credentials_ec2.c | 39 +++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/aws/flb_aws_imds.h b/include/fluent-bit/aws/flb_aws_imds.h index b18e0192e6e..5203fdd4f68 100644 --- a/include/fluent-bit/aws/flb_aws_imds.h +++ b/include/fluent-bit/aws/flb_aws_imds.h @@ -25,6 +25,9 @@ #define FLB_AWS_IMDS_PORT 80 #define FLB_AWS_IMDS_TIMEOUT 1 /* 1 second */ +/* Environment variable for custom IMDS endpoint */ +#define AWS_EC2_METADATA_SERVICE_ENDPOINT_ENV "AWS_EC2_METADATA_SERVICE_ENDPOINT" + #define FLB_AWS_IMDS_VERSION_EVALUATE 0 #define FLB_AWS_IMDS_VERSION_1 1 #define FLB_AWS_IMDS_VERSION_2 2 diff --git a/src/aws/flb_aws_credentials_ec2.c b/src/aws/flb_aws_credentials_ec2.c index d3cc2a4afaa..4139c43e92f 100644 --- a/src/aws/flb_aws_credentials_ec2.c +++ b/src/aws/flb_aws_credentials_ec2.c @@ -232,6 +232,14 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, struct flb_aws_provider_ec2 *implementation; struct flb_aws_provider *provider; struct flb_upstream *upstream; + char *endpoint; + flb_sds_t host = NULL; + flb_sds_t port_str = NULL; + flb_sds_t protocol = NULL; + flb_sds_t path = NULL; + const char *use_host; + int use_port; + int ret; provider = flb_calloc(1, sizeof(struct flb_aws_provider)); @@ -253,8 +261,35 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, provider->provider_vtable = &ec2_provider_vtable; provider->implementation = implementation; - upstream = flb_upstream_create(config, FLB_AWS_IMDS_HOST, FLB_AWS_IMDS_PORT, - FLB_IO_TCP, NULL); + /* Check for custom IMDS endpoint */ + use_host = FLB_AWS_IMDS_HOST; + use_port = FLB_AWS_IMDS_PORT; + + endpoint = getenv(AWS_EC2_METADATA_SERVICE_ENDPOINT_ENV); + if (endpoint && strlen(endpoint) > 0) { + ret = flb_utils_url_split_sds(endpoint, &protocol, &host, &port_str, &path); + if (ret >= 0 && host) { + use_host = host; + if (port_str) { + use_port = atoi(port_str); + if (use_port <= 0 || use_port > 65535) { + use_port = FLB_AWS_IMDS_PORT; + } + } + flb_info("[aws_credentials] Using custom IMDS endpoint: %s:%d", + use_host, use_port); + } + flb_sds_destroy(protocol); + flb_sds_destroy(port_str); + flb_sds_destroy(path); + } + + upstream = flb_upstream_create(config, use_host, use_port, FLB_IO_TCP, NULL); + + if (host) { + flb_sds_destroy(host); + } + if (!upstream) { flb_aws_provider_destroy(provider); flb_debug("[aws_credentials] unable to connect to EC2 IMDS."); @@ -278,7 +313,7 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, implementation->client->provider = NULL; implementation->client->region = NULL; implementation->client->service = NULL; - implementation->client->port = 80; + implementation->client->port = use_port; implementation->client->flags = 0; implementation->client->proxy = NULL; implementation->client->upstream = upstream; diff --git a/tests/internal/aws_credentials_ec2.c b/tests/internal/aws_credentials_ec2.c index 6370b1561e9..e89e56d20be 100644 --- a/tests/internal/aws_credentials_ec2.c +++ b/tests/internal/aws_credentials_ec2.c @@ -1025,6 +1025,44 @@ static void test_ec2_imds_create_and_destroy() flb_config_exit(config_fluent); } +static void test_ec2_provider_custom_endpoint() +{ + setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", "http://127.0.0.1:9911", 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/api/token"), + expect(METHOD, FLB_HTTP_PUT), + set(STATUS, 200), + set(PAYLOAD, "TESTTOKEN") + ), + response( + expect(URI, "/latest/meta-data/iam/security-credentials/"), + expect(HEADER, "X-aws-ec2-metadata-token", "TESTTOKEN"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "test-role") + ), + response( + expect(URI, "/latest/meta-data/iam/security-credentials/test-role"), + expect(HEADER, "X-aws-ec2-metadata-token", "TESTTOKEN"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "{\"AccessKeyId\":\"AKIATEST\",\"SecretAccessKey\":\"SECRET\",\"Token\":\"TOKEN\"}") + ) + )); + + creds = provider->provider_vtable->get_credentials(provider); + TEST_CHECK(creds != NULL); + TEST_CHECK(strcmp("AKIATEST", creds->access_key_id) == 0); + TEST_CHECK(strcmp("SECRET", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("TOKEN", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + unsetenv("AWS_EC2_METADATA_SERVICE_ENDPOINT"); + cleanup_test(); +} + TEST_LIST = { { "test_ec2_provider_v2" , test_ec2_provider_v2}, { "test_ec2_provider_v1" , test_ec2_provider_v1}, @@ -1033,5 +1071,6 @@ TEST_LIST = { { "test_ec2_provider_acquire_token_error" , test_ec2_provider_acquire_token_error}, { "test_ec2_provider_metadata_request_error" , test_ec2_provider_metadata_request_error}, { "test_ec2_imds_create_and_destroy" , test_ec2_imds_create_and_destroy}, + { "test_ec2_provider_custom_endpoint" , test_ec2_provider_custom_endpoint}, { 0 } }; From a981de42e81d9fa29f51fb991cdd0bd2b5f61542 Mon Sep 17 00:00:00 2001 From: drappier-charles Date: Sun, 2 Nov 2025 22:17:57 +0100 Subject: [PATCH 2/4] Refactor IMDS client creation to support custom endpoints - Removed hardcoded checks for the EC2 IMDS TCP host and port in the `flb_aws_imds_create` function. - Added comments to clarify that custom IMDS endpoints can now be specified via the `AWS_EC2_METADATA_SERVICE_ENDPOINT` environment variable. These changes enhance flexibility for users needing to connect to non-default IMDS endpoints, following the recent addition of the environment variable support. --- src/aws/flb_aws_imds.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/aws/flb_aws_imds.c b/src/aws/flb_aws_imds.c index be4924b4c2c..11310f69e19 100644 --- a/src/aws/flb_aws_imds.c +++ b/src/aws/flb_aws_imds.c @@ -80,17 +80,9 @@ struct flb_aws_imds *flb_aws_imds_create(const struct flb_aws_imds_config *imds_ flb_aws_imds_destroy(ctx); return NULL; } - if (0 != strncmp(ec2_imds_client->upstream->tcp_host, FLB_AWS_IMDS_HOST, - FLB_AWS_IMDS_HOST_LEN)) { - flb_debug("[imds] ec2_imds_client tcp host must be set to %s", FLB_AWS_IMDS_HOST); - flb_aws_imds_destroy(ctx); - return NULL; - } - if (ec2_imds_client->upstream->tcp_port != FLB_AWS_IMDS_PORT) { - flb_debug("[imds] ec2_imds_client tcp port must be set to %i", FLB_AWS_IMDS_PORT); - flb_aws_imds_destroy(ctx); - return NULL; - } + + /* Allow custom IMDS endpoints via AWS_EC2_METADATA_SERVICE_ENDPOINT */ + /* The hardcoded host/port checks have been removed to support custom endpoints */ /* Connect client */ ctx->ec2_imds_client = ec2_imds_client; From 8601506acb284377573425a195ac6f3cc8bd4202 Mon Sep 17 00:00:00 2001 From: drappier-charles Date: Sun, 2 Nov 2025 22:30:49 +0100 Subject: [PATCH 3/4] Enhance IMDS timeout handling for custom endpoints - Introduced a new constant `FLB_AWS_IMDS_TIMEOUT_CUSTOM` to specify a 10-second timeout for custom IMDS endpoints, accommodating longer connection times for services like IAM Roles Anywhere. - Updated the EC2 provider creation logic to dynamically set connection and I/O timeouts based on whether a custom endpoint is used. - Added informative logging to indicate when the extended timeout is applied. These changes improve the flexibility and reliability of connections to custom IMDS endpoints, ensuring appropriate timeout settings are utilized. --- include/fluent-bit/aws/flb_aws_imds.h | 3 ++- src/aws/flb_aws_credentials_ec2.c | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/include/fluent-bit/aws/flb_aws_imds.h b/include/fluent-bit/aws/flb_aws_imds.h index 5203fdd4f68..e349b337e31 100644 --- a/include/fluent-bit/aws/flb_aws_imds.h +++ b/include/fluent-bit/aws/flb_aws_imds.h @@ -23,7 +23,8 @@ #define FLB_AWS_IMDS_HOST "169.254.169.254" #define FLB_AWS_IMDS_HOST_LEN 15 #define FLB_AWS_IMDS_PORT 80 -#define FLB_AWS_IMDS_TIMEOUT 1 /* 1 second */ +#define FLB_AWS_IMDS_TIMEOUT 1 /* 1 second - for standard AWS IMDS */ +#define FLB_AWS_IMDS_TIMEOUT_CUSTOM 10 /* 10 seconds - for custom IMDS endpoints like IAM Roles Anywhere */ /* Environment variable for custom IMDS endpoint */ #define AWS_EC2_METADATA_SERVICE_ENDPOINT_ENV "AWS_EC2_METADATA_SERVICE_ENDPOINT" diff --git a/src/aws/flb_aws_credentials_ec2.c b/src/aws/flb_aws_credentials_ec2.c index 4139c43e92f..59c6faa6385 100644 --- a/src/aws/flb_aws_credentials_ec2.c +++ b/src/aws/flb_aws_credentials_ec2.c @@ -264,12 +264,14 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, /* Check for custom IMDS endpoint */ use_host = FLB_AWS_IMDS_HOST; use_port = FLB_AWS_IMDS_PORT; + int use_custom_endpoint = FLB_FALSE; endpoint = getenv(AWS_EC2_METADATA_SERVICE_ENDPOINT_ENV); if (endpoint && strlen(endpoint) > 0) { ret = flb_utils_url_split_sds(endpoint, &protocol, &host, &port_str, &path); if (ret >= 0 && host) { use_host = host; + use_custom_endpoint = FLB_TRUE; if (port_str) { use_port = atoi(port_str); if (use_port <= 0 || use_port > 65535) { @@ -296,9 +298,20 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config, return NULL; } - /* IMDSv2 token request will timeout if hops = 1 and running within container */ - upstream->base.net.connect_timeout = FLB_AWS_IMDS_TIMEOUT; - upstream->base.net.io_timeout = FLB_AWS_IMDS_TIMEOUT; + /* + * Set timeout based on endpoint type: + * - Standard AWS IMDS: 1 second (fast local endpoint) + * - Custom IMDS endpoints (e.g., IAM Roles Anywhere): 10 seconds (certificate auth takes longer) + */ + if (use_custom_endpoint) { + upstream->base.net.connect_timeout = FLB_AWS_IMDS_TIMEOUT_CUSTOM; + upstream->base.net.io_timeout = FLB_AWS_IMDS_TIMEOUT_CUSTOM; + flb_info("[aws_credentials] Using extended timeout (%d seconds) for custom IMDS endpoint", + FLB_AWS_IMDS_TIMEOUT_CUSTOM); + } else { + upstream->base.net.connect_timeout = FLB_AWS_IMDS_TIMEOUT; + upstream->base.net.io_timeout = FLB_AWS_IMDS_TIMEOUT; + } upstream->base.net.keepalive = FLB_FALSE; /* On timeout, the connection is broken */ implementation->client = generator->create(); From 15e973f861813f20c21ab040d39260c30a4713be Mon Sep 17 00:00:00 2001 From: drappier-charles Date: Sun, 2 Nov 2025 22:39:34 +0100 Subject: [PATCH 4/4] Refactor IMDS version retrieval logic for improved compatibility - Updated the `get_imds_version` function to first attempt to retrieve an IMDSv2 token, falling back to IMDSv1 if the request fails. This change enhances compatibility with custom IMDS implementations, such as IAM Roles Anywhere. - Removed the previous method of sending an invalid token header to determine IMDSv2 availability, streamlining the logic and improving clarity. - Added debug and info logging to provide better insights during the token retrieval process. These changes improve the robustness of the IMDS version detection mechanism, ensuring a more reliable connection to metadata services. --- src/aws/flb_aws_imds.c | 73 +++++++++++++----------------------------- 1 file changed, 22 insertions(+), 51 deletions(-) diff --git a/src/aws/flb_aws_imds.c b/src/aws/flb_aws_imds.c index 11310f69e19..a9cf93b0211 100644 --- a/src/aws/flb_aws_imds.c +++ b/src/aws/flb_aws_imds.c @@ -242,7 +242,6 @@ static int get_imds_version(struct flb_aws_imds *ctx) { int ret; struct flb_aws_client *client = ctx->ec2_imds_client; - struct flb_aws_header invalid_token_header; struct flb_http_client *c = NULL; if (ctx->imds_version != FLB_AWS_IMDS_VERSION_EVALUATE) { @@ -251,65 +250,37 @@ static int get_imds_version(struct flb_aws_imds *ctx) /* * Evaluate version - * To evaluate wether IMDSv2 is available, send an invalid token - * in IMDS request. If response status is 'Unauthorized', then IMDSv2 - * is available. + * Try to get an IMDSv2 token first. If that fails, fall back to IMDSv1. + * This approach is more compatible with custom IMDS implementations like IAM Roles Anywhere. */ - invalid_token_header = imds_v2_token_token_header_template; - invalid_token_header.val = "INVALID"; - invalid_token_header.val_len = 7; - c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, NULL, 0, - &invalid_token_header, 1); - + ctx->imds_version = FLB_AWS_IMDS_VERSION_2; + ret = refresh_imds_v2_token(ctx); + if (ret == 0) { + /* Successfully got IMDSv2 token */ + flb_info("[imds] using IMDSv2"); + return FLB_AWS_IMDS_VERSION_2; + } + + /* IMDSv2 token request failed, try IMDSv1 */ + flb_debug("[imds] IMDSv2 token request failed, testing IMDSv1"); + ctx->imds_version = FLB_AWS_IMDS_VERSION_EVALUATE; + c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, + NULL, 0, NULL, 0); + if (!c) { flb_debug("[imds] imds endpoint unavailable"); return FLB_AWS_IMDS_VERSION_EVALUATE; } - - /* Unauthorized response means that IMDS version 2 is in use */ - if (c->resp.status == 401) { - ctx->imds_version = FLB_AWS_IMDS_VERSION_2; - ret = refresh_imds_v2_token(ctx); - if (ret == -1) { - /* - * Token cannot be refreshed, test IMDSv1 - * If IMDSv1 cannot be used, response will be status 401 - */ - flb_http_client_destroy(c); - ctx->imds_version = FLB_AWS_IMDS_VERSION_EVALUATE; - c = client->client_vtable->request(client, FLB_HTTP_GET, FLB_AWS_IMDS_ROOT, - NULL, 0, NULL, 0); - if (!c) { - flb_debug("[imds] imds v1 attempt, endpoint unavailable"); - return FLB_AWS_IMDS_VERSION_EVALUATE; - } - - if (c->resp.status == 200) { - flb_info("[imds] to use IMDSv2, set --http-put-response-hop-limit to 2"); - } - else { - /* IMDSv1 unavailable. IMDSv2 beyond network hop count */ - flb_warn("[imds] failed to retrieve IMDSv2 token and IMDSv1 unavailable. " - "This is likely due to instance-metadata-options " - "--http-put-response-hop-limit being set to 1 and --http-tokens " - "set to required. " - "To use IMDSv2, please set --http-put-response-hop-limit to 2 as " - "described https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/" - "configuring-instance-metadata-options.html"); - } - } - } - - /* - * Success means that IMDS version 1 is in use - */ + if (c->resp.status == 200) { - flb_warn("[imds] falling back on IMDSv1"); + flb_info("[imds] falling back to IMDSv1"); ctx->imds_version = FLB_AWS_IMDS_VERSION_1; + flb_http_client_destroy(c); + return FLB_AWS_IMDS_VERSION_1; } - + flb_http_client_destroy(c); - return ctx->imds_version; + return FLB_AWS_IMDS_VERSION_EVALUATE; } /*