Skip to content

aws_credentials: fix use-after-free when a refresh races get_credentials() #12207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Nahid-NHB wants to merge 2 commits into
base: master
Choose a base branch
from
Open

aws_credentials: fix use-after-free when a refresh races get_credentials() #12207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d96fb2
aws: copy cached credentials under a lock
Nahid-NHB Aug 3, 2026
1492bd8
tests: internal: cover concurrent credential refresh
Nahid-NHB Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 46 additions & 5 deletions include/fluent-bit/flb_aws_credentials.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,18 @@ struct flb_aws_provider {
*/
pthread_mutex_t lock;

/*
* Protects the credentials cached by the provider implementation.
*
* The lock above is a trylock because it is held across the network call
* that fetches new credentials; a reader cannot take it without risking a
* deadlock between two coroutines running on the same thread. This lock is
* only ever held for the pointer swap that publishes new credentials and
* for the copy that readers make, so it never spans a yield point and can
* be taken with a blocking lock.
*/
pthread_mutex_t cache_lock;

struct flb_aws_provider_vtable *provider_vtable;

void *implementation;
Expand Down Expand Up @@ -338,19 +350,48 @@ int exec_credential_process(char* process, struct flb_aws_credentials** creds,
#endif /* FLB_HAVE_AWS_CREDENTIAL_PROCESS */

/*
* Fluent Bit is single-threaded but asynchonous. Only one co-routine will
* be running at a time, and they only pause/resume for IO.
* A provider is shared by every flush thread of an output with `workers` set,
* so the cached credentials are read and replaced concurrently by real threads.
*
* Thus, while synchronization is needed (to prevent multiple co-routines
* from duplicating effort and performing the same work), it can be obtained
* using a simple integer flag on the provider.
* The provider lock deduplicates refreshes: only the coroutine that wins it
* calls out to the credentials endpoint. It is a trylock because it is held
* across that network call, which yields the coroutine.
*
* The cache lock protects the cached credentials themselves. Use the
* flb_aws_cache_* helpers below instead of touching the cached pointer
* directly; they hold the cache lock for the copy and for the swap, so a
* reader can never end up with a pointer that the refresh path has freed.
*/

/* Like a traditional try lock- it does not block if the lock is not obtained */
int try_lock_provider(struct flb_aws_provider *provider);

void unlock_provider(struct flb_aws_provider *provider);

/*
* Returns a copy of the credentials in *cache, or NULL if the cache is empty
* or the copy could not be allocated. The caller owns the returned copy.
*/
struct flb_aws_credentials *flb_aws_cache_get_credentials(struct flb_aws_provider
*provider,
struct flb_aws_credentials
**cache);

/* Returns the refresh deadline stored in *next_refresh */
time_t flb_aws_cache_get_refresh_time(struct flb_aws_provider *provider,
time_t *next_refresh);

/*
* Publishes creds as the new contents of *cache and refresh_time as the new
* contents of *next_refresh. Takes ownership of creds and frees whatever was
* cached before.
*/
void flb_aws_cache_set_credentials(struct flb_aws_provider *provider,
struct flb_aws_credentials **cache,
struct flb_aws_credentials *creds,
time_t *next_refresh,
time_t refresh_time);


/*
* HTTP Credentials Provider - retrieve credentials from a local http server
Expand Down
104 changes: 104 additions & 0 deletions src/aws/flb_aws_credentials.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,7 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config
}

pthread_mutex_init(&provider->lock, NULL);
pthread_mutex_init(&provider->cache_lock, NULL);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

implementation = flb_calloc(1, sizeof(struct flb_aws_provider_chain));

Expand Down Expand Up @@ -774,6 +775,7 @@ void flb_aws_provider_destroy(struct flb_aws_provider *provider)
}

pthread_mutex_destroy(&provider->lock);
pthread_mutex_destroy(&provider->cache_lock);

/* free managed dependencies */
if (provider->base_aws_provider) {
Expand Down Expand Up @@ -863,3 +865,105 @@ void unlock_provider(struct flb_aws_provider *provider)
{
pthread_mutex_unlock(&provider->lock);
}

/*
* The cache lock is never held across IO, so unlike the provider lock it is
* safe to block on it.
*/
static inline void lock_provider_cache(struct flb_aws_provider *provider)
{
pthread_mutex_lock(&provider->cache_lock);
}

static inline void unlock_provider_cache(struct flb_aws_provider *provider)
{
pthread_mutex_unlock(&provider->cache_lock);
}

struct flb_aws_credentials *flb_aws_cache_get_credentials(struct flb_aws_provider
*provider,
struct flb_aws_credentials
**cache)
{
struct flb_aws_credentials *cached;
struct flb_aws_credentials *creds = NULL;

lock_provider_cache(provider);

cached = *cache;
if (!cached) {
unlock_provider_cache(provider);
return NULL;
}

creds = flb_calloc(1, sizeof(struct flb_aws_credentials));
if (!creds) {
goto error;
}

creds->access_key_id = flb_sds_create(cached->access_key_id);
if (!creds->access_key_id) {
goto error;
}

creds->secret_access_key = flb_sds_create(cached->secret_access_key);
if (!creds->secret_access_key) {
goto error;
}

if (cached->session_token) {
creds->session_token = flb_sds_create(cached->session_token);
if (!creds->session_token) {
goto error;
}
}
else {
creds->session_token = NULL;
}

unlock_provider_cache(provider);
return creds;

error:
unlock_provider_cache(provider);
flb_errno();
flb_aws_credentials_destroy(creds);
return NULL;
}

time_t flb_aws_cache_get_refresh_time(struct flb_aws_provider *provider,
time_t *next_refresh)
{
time_t refresh_time;

lock_provider_cache(provider);
refresh_time = *next_refresh;
unlock_provider_cache(provider);

return refresh_time;
}

void flb_aws_cache_set_credentials(struct flb_aws_provider *provider,
struct flb_aws_credentials **cache,
struct flb_aws_credentials *creds,
time_t *next_refresh,
time_t refresh_time)
{
struct flb_aws_credentials *previous;

lock_provider_cache(provider);

previous = *cache;
*cache = creds;
*next_refresh = refresh_time;

unlock_provider_cache(provider);

/*
* Free the old credentials after the swap and outside of the lock.
* Readers hold the cache lock for the whole copy, so none of them was
* halfway through reading these when we swapped them out, and anyone
* arriving now sees the new ones.
*/
flb_aws_credentials_destroy(previous);
}
86 changes: 32 additions & 54 deletions src/aws/flb_aws_credentials_ec2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@
#define AWS_IMDS_ROLE_PATH_LEN 43

struct flb_aws_provider_ec2;
static int get_creds_ec2(struct flb_aws_provider_ec2 *implementation);
static int ec2_credentials_request(struct flb_aws_provider_ec2
*implementation, char *cred_path);
static int get_creds_ec2(struct flb_aws_provider *provider,
struct flb_aws_provider_ec2 *implementation);
static int ec2_credentials_request(struct flb_aws_provider *provider,
struct flb_aws_provider_ec2 *implementation,
char *cred_path);

/* EC2 IMDS Provider */

Expand All @@ -58,25 +60,34 @@ struct flb_aws_credentials *get_credentials_fn_ec2(struct flb_aws_provider
*provider)
{
struct flb_aws_credentials *creds;
time_t next_refresh;
int refresh = FLB_FALSE;
struct flb_aws_provider_ec2 *implementation = provider->implementation;

flb_debug("[aws_credentials] Requesting credentials from the "
"EC2 provider..");

next_refresh = flb_aws_cache_get_refresh_time(provider,
&implementation->next_refresh);

/* a negative next_refresh means that auto-refresh is disabled */
if (implementation->next_refresh > 0
&& time(NULL) > implementation->next_refresh) {
if (next_refresh > 0 && time(NULL) > next_refresh) {
refresh = FLB_TRUE;
}
if (!implementation->creds || refresh == FLB_TRUE) {

creds = flb_aws_cache_get_credentials(provider, &implementation->creds);
if (!creds || refresh == FLB_TRUE) {
if (try_lock_provider(provider)) {
get_creds_ec2(implementation);
get_creds_ec2(provider, implementation);
unlock_provider(provider);

flb_aws_credentials_destroy(creds);
creds = flb_aws_cache_get_credentials(provider,
&implementation->creds);
}
}

if (!implementation->creds) {
if (!creds) {
/*
* We failed to lock the provider and creds are unset. This means that
* another co-routine is performing the refresh.
Expand All @@ -88,40 +99,6 @@ struct flb_aws_credentials *get_credentials_fn_ec2(struct flb_aws_provider
return NULL;
}

creds = flb_calloc(1, sizeof(struct flb_aws_credentials));
if (!creds) {
flb_errno();
return NULL;
}

creds->access_key_id = flb_sds_create(implementation->creds->access_key_id);
if (!creds->access_key_id) {
flb_errno();
flb_aws_credentials_destroy(creds);
return NULL;
}

creds->secret_access_key = flb_sds_create(implementation->creds->
secret_access_key);
if (!creds->secret_access_key) {
flb_errno();
flb_aws_credentials_destroy(creds);
return NULL;
}

if (implementation->creds->session_token) {
creds->session_token = flb_sds_create(implementation->creds->
session_token);
if (!creds->session_token) {
flb_errno();
flb_aws_credentials_destroy(creds);
return NULL;
}

} else {
creds->session_token = NULL;
}

return creds;
}

Expand All @@ -131,7 +108,7 @@ int refresh_fn_ec2(struct flb_aws_provider *provider) {

flb_debug("[aws_credentials] Refresh called on the EC2 IMDS provider");
if (try_lock_provider(provider)) {
ret = get_creds_ec2(implementation);
ret = get_creds_ec2(provider, implementation);
unlock_provider(provider);
}
return ret;
Expand All @@ -145,7 +122,7 @@ int init_fn_ec2(struct flb_aws_provider *provider) {

flb_debug("[aws_credentials] Init called on the EC2 IMDS provider");
if (try_lock_provider(provider)) {
ret = get_creds_ec2(implementation);
ret = get_creds_ec2(provider, implementation);
unlock_provider(provider);
}

Expand Down Expand Up @@ -241,6 +218,7 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config,
}

pthread_mutex_init(&provider->lock, NULL);
pthread_mutex_init(&provider->cache_lock, NULL);

implementation = flb_calloc(1, sizeof(struct flb_aws_provider_ec2));

Expand Down Expand Up @@ -296,7 +274,8 @@ struct flb_aws_provider *flb_ec2_provider_create(struct flb_config *config,
}

/* Requests creds from IMDSv1 and sets them on the provider */
static int get_creds_ec2(struct flb_aws_provider_ec2 *implementation)
static int get_creds_ec2(struct flb_aws_provider *provider,
struct flb_aws_provider_ec2 *implementation)
{
int ret;
flb_sds_t instance_role;
Expand Down Expand Up @@ -337,16 +316,17 @@ static int get_creds_ec2(struct flb_aws_provider_ec2 *implementation)
}

/* request creds */
ret = ec2_credentials_request(implementation, cred_path);
ret = ec2_credentials_request(provider, implementation, cred_path);

flb_sds_destroy(instance_role);
flb_free(cred_path);
return ret;

}

static int ec2_credentials_request(struct flb_aws_provider_ec2
*implementation, char *cred_path)
static int ec2_credentials_request(struct flb_aws_provider *provider,
struct flb_aws_provider_ec2 *implementation,
char *cred_path)
{
int ret;
flb_sds_t credentials_response;
Expand All @@ -370,12 +350,10 @@ static int ec2_credentials_request(struct flb_aws_provider_ec2
return -1;
}

/* destroy existing credentials first */
flb_aws_credentials_destroy(implementation->creds);
implementation->creds = NULL;
/* set new creds */
implementation->creds = creds;
implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW;
/* publish the new credentials; the old ones are freed for us */
flb_aws_cache_set_credentials(provider, &implementation->creds, creds,
&implementation->next_refresh,
expiration - FLB_AWS_REFRESH_WINDOW);

flb_sds_destroy(credentials_response);
return 0;
Expand Down
Loading
Loading