diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 5b30d8bae4a..07efcf8c745 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -738,8 +738,13 @@ static int create_blob(struct flb_azure_blob *ctx, const char *path_prefix, char } /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_TRUE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + ret = azb_http_client_setup(ctx, c, -1, FLB_TRUE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + status = FLB_RETRY; + goto cleanup_create; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -817,8 +822,13 @@ static int delete_blob(struct flb_azure_blob *ctx, } /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_TRUE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + ret = azb_http_client_setup(ctx, c, -1, FLB_TRUE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + status = FLB_RETRY; + goto cleanup_delete; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -1000,8 +1010,17 @@ static int http_send_blob(struct flb_config *config, struct flb_azure_blob *ctx, } /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, (ssize_t) payload_size, FLB_FALSE, - content_type, content_encoding); + ret = azb_http_client_setup(ctx, c, (ssize_t) payload_size, FLB_FALSE, + content_type, content_encoding); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + if (compressed == FLB_TRUE) { + flb_free(payload_buf); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -1228,13 +1247,21 @@ static int create_container(struct flb_azure_blob *ctx, char *name) NULL, 0, NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context"); + flb_sds_destroy(uri); flb_upstream_conn_release(u_conn); return FLB_FALSE; } /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + ret = azb_http_client_setup(ctx, c, -1, FLB_FALSE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + return FLB_FALSE; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -1320,14 +1347,22 @@ static int ensure_container(struct flb_azure_blob *ctx) NULL, 0, NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context"); + flb_sds_destroy(uri); flb_upstream_conn_release(u_conn); return FLB_FALSE; } flb_http_strip_port_from_host(c); /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + ret = azb_http_client_setup(ctx, c, -1, FLB_FALSE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + return FLB_FALSE; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -1335,6 +1370,7 @@ static int ensure_container(struct flb_azure_blob *ctx) if (ret == -1) { flb_plg_error(ctx->ins, "error requesting container properties"); + flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); return FLB_FALSE; } @@ -1372,6 +1408,7 @@ static int ensure_container(struct flb_azure_blob *ctx) static int cb_azure_blob_init(struct flb_output_instance *ins, struct flb_config *config, void *data) { + int ret; struct flb_azure_blob *ctx = NULL; (void) ins; (void) config; @@ -1384,8 +1421,9 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, return -1; } + ctx->ins = ins; + if (ctx->buffering_enabled == FLB_TRUE) { - ctx->ins = ins; ctx->retry_time = 0; /* Initialize local storage */ @@ -1399,14 +1437,17 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, /* validate 'total_file_size' */ if (ctx->file_size <= 0) { flb_plg_error(ctx->ins, "Failed to parse upload_file_size"); + azure_blob_store_exit(ctx); return -1; } if (ctx->file_size < 1000000) { flb_plg_error(ctx->ins, "upload_file_size must be at least 1MB"); + azure_blob_store_exit(ctx); return -1; } if (ctx->file_size > MAX_FILE_SIZE) { flb_plg_error(ctx->ins, "Max total_file_size must be lower than %ld bytes", MAX_FILE_SIZE); + azure_blob_store_exit(ctx); return -1; } ctx->has_old_buffers = azure_blob_store_has_data(ctx); @@ -1415,6 +1456,59 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, flb_plg_info(ctx->ins, "Using upload size %lu bytes", ctx->file_size); } + /* Initialize OAuth2 context for service principal auth */ + if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL) { + ret = pthread_mutex_init(&ctx->token_mutex, NULL); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to initialize token mutex"); + if (ctx->buffering_enabled == FLB_TRUE) { + azure_blob_store_exit(ctx); + } + flb_azure_blob_conf_destroy(ctx); + return -1; + } + + flb_sds_t token_url; + + token_url = flb_sds_create_size(256); + if (!token_url) { + flb_plg_error(ctx->ins, "failed to allocate token URL"); + pthread_mutex_destroy(&ctx->token_mutex); + if (ctx->buffering_enabled == FLB_TRUE) { + azure_blob_store_exit(ctx); + } + flb_azure_blob_conf_destroy(ctx); + return -1; + } + + ret = flb_sds_snprintf(&token_url, flb_sds_alloc(token_url), + "%s/%s/oauth2/v2.0/token", + AZURE_BLOB_DEFAULT_AUTHORITY_HOST, ctx->tenant_id); + if (ret < 0) { + flb_plg_error(ctx->ins, "failed to build token URL"); + flb_sds_destroy(token_url); + pthread_mutex_destroy(&ctx->token_mutex); + if (ctx->buffering_enabled == FLB_TRUE) { + azure_blob_store_exit(ctx); + } + flb_azure_blob_conf_destroy(ctx); + return -1; + } + + ctx->o = flb_oauth2_create(ctx->config, token_url, AZURE_BLOB_TOKEN_REFRESH); + flb_sds_destroy(token_url); + + if (!ctx->o) { + flb_plg_error(ctx->ins, "failed to create OAuth2 context"); + pthread_mutex_destroy(&ctx->token_mutex); + if (ctx->buffering_enabled == FLB_TRUE) { + azure_blob_store_exit(ctx); + } + flb_azure_blob_conf_destroy(ctx); + return -1; + } + } + flb_output_set_context(ins, ctx); flb_output_set_http_debug_callbacks(ins); @@ -1691,7 +1785,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context const char *commit_prefix = azb_commit_prefix_with_fallback(ctx, path_prefix); ret = azb_block_blob_commit_file_parts(ctx, file_id, file_path, part_ids, commit_prefix); - if (ret == -1) { + if (ret != FLB_OK) { flb_plg_error(ctx->ins, "cannot commit blob file parts for file id=%" PRIu64 " path=%s", file_id, file_path); } @@ -2390,6 +2484,11 @@ static int cb_azure_blob_exit(void *data, struct flb_config *config) ctx->u = NULL; } + /* Destroy token mutex for service principal auth */ + if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL) { + pthread_mutex_destroy(&ctx->token_mutex); + } + flb_azure_blob_conf_destroy(ctx); return 0; } @@ -2514,7 +2613,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "auth_type", "key", 0, FLB_TRUE, offsetof(struct flb_azure_blob, auth_type), - "Set the auth type: key or sas" + "Set the auth type: key, sas, or service_principal" }, { @@ -2523,6 +2622,24 @@ static struct flb_config_map config_map[] = { "Azure Blob SAS token" }, + { + FLB_CONFIG_MAP_STR, "tenant_id", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, tenant_id), + "Azure AD tenant ID (required for service_principal auth)" + }, + + { + FLB_CONFIG_MAP_STR, "client_id", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, client_id), + "Azure AD client ID (required for service_principal auth)" + }, + + { + FLB_CONFIG_MAP_STR, "client_secret", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, client_secret), + "Azure AD client secret (required for service_principal auth)" + }, + { FLB_CONFIG_MAP_STR, "database_file", NULL, 0, FLB_TRUE, offsetof(struct flb_azure_blob, database_file), diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index e79d13945dd..8d4a14cf766 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -25,6 +25,7 @@ #include #include #include +#include /* Content-Type */ #define AZURE_BLOB_CT "Content-Type" @@ -53,6 +54,12 @@ #define AZURE_BLOB_AUTH_KEY 0 #define AZURE_BLOB_AUTH_SAS 1 +#define AZURE_BLOB_AUTH_SERVICE_PRINCIPAL 2 + +/* OAuth2 defaults for service principal authentication */ +#define AZURE_BLOB_DEFAULT_AUTHORITY_HOST "https://login.microsoftonline.com" +#define AZURE_BLOB_OAUTH_SCOPE "https://storage.azure.com/.default" +#define AZURE_BLOB_TOKEN_REFRESH 3000 /* refresh token every 50 minutes */ struct flb_azure_blob { int auto_create_container; @@ -69,6 +76,11 @@ struct flb_azure_blob { flb_sds_t date_key; flb_sds_t auth_type; flb_sds_t sas_token; + + /* Service Principal authentication fields */ + flb_sds_t tenant_id; + flb_sds_t client_id; + flb_sds_t client_secret; flb_sds_t database_file; size_t part_size; time_t upload_parts_timeout; @@ -125,6 +137,10 @@ struct flb_azure_blob { unsigned char *decoded_sk; /* decoded shared key */ size_t decoded_sk_size; /* size of decoded shared key */ + /* Service Principal OAuth2 context */ + struct flb_oauth2 *o; + pthread_mutex_t token_mutex; + #ifdef FLB_HAVE_SQLDB /* * SQLite by default is not built with multi-threading enabled, and diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index 356c0ad24d6..bccbb3872fc 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -310,9 +310,15 @@ int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb } /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, flb_sds_len(payload), - FLB_FALSE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + ret = azb_http_client_setup(ctx, c, flb_sds_len(payload), + FLB_FALSE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to setup HTTP client"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } /* Send HTTP request */ ret = flb_http_do(c, &b_sent); @@ -320,6 +326,8 @@ int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb /* Validate HTTP status */ if (ret == -1) { flb_plg_error(ctx->ins, "error sending block_blob"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); return FLB_RETRY; } diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index 9c2d2300d7b..205df49c7fc 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "azure_blob.h" #include "azure_blob_conf.h" @@ -573,6 +574,27 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } + /* Set Auth type - must be parsed before remote configuration */ + tmp = (char *) flb_output_get_property("auth_type", ins); + if (!tmp) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else { + if (strcasecmp(tmp, "key") == 0) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else if (strcasecmp(tmp, "sas") == 0) { + ctx->atype = AZURE_BLOB_AUTH_SAS; + } + else if (strcasecmp(tmp, "service_principal") == 0) { + ctx->atype = AZURE_BLOB_AUTH_SERVICE_PRINCIPAL; + } + else { + flb_plg_error(ctx->ins, "invalid auth_type value '%s'", tmp); + return NULL; + } + } + if (ctx->configuration_endpoint_url != NULL) { ret = flb_azure_blob_apply_remote_configuration(ctx); @@ -604,23 +626,7 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } - /* Set Auth type */ - tmp = (char *) flb_output_get_property("auth_type", ins); - if (!tmp) { - ctx->atype = AZURE_BLOB_AUTH_KEY; - } - else { - if (strcasecmp(tmp, "key") == 0) { - ctx->atype = AZURE_BLOB_AUTH_KEY; - } - else if (strcasecmp(tmp, "sas") == 0) { - ctx->atype = AZURE_BLOB_AUTH_SAS; - } - else { - flb_plg_error(ctx->ins, "invalid auth_type value '%s'", tmp); - return NULL; - } - } + /* Validate auth-specific required fields */ if (ctx->atype == AZURE_BLOB_AUTH_KEY && ctx->shared_key == NULL) { flb_plg_error(ctx->ins, "'shared_key' has not been set"); @@ -637,6 +643,21 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } } + if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL) { + if (ctx->tenant_id == NULL || flb_sds_len(ctx->tenant_id) == 0) { + flb_plg_error(ctx->ins, "'tenant_id' is required for service_principal auth"); + return NULL; + } + if (ctx->client_id == NULL || flb_sds_len(ctx->client_id) == 0) { + flb_plg_error(ctx->ins, "'client_id' is required for service_principal auth"); + return NULL; + } + if (ctx->client_secret == NULL || flb_sds_len(ctx->client_secret) == 0) { + flb_plg_error(ctx->ins, "'client_secret' is required for service_principal auth"); + return NULL; + } + } + /* If the shared key is set decode it */ if (ctx->atype == AZURE_BLOB_AUTH_KEY && ctx->shared_key != NULL) { @@ -696,6 +717,17 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } + /* + * Service principal authentication requires TLS for Azure Storage API + * requests. Validate that TLS is explicitly enabled. + */ + if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL && + ins->use_tls != FLB_TRUE) { + flb_plg_error(ctx->ins, + "service_principal auth requires TLS; set 'tls On'"); + return NULL; + } + /* * Setting up the real endpoint: * @@ -795,7 +827,8 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob" : "blockblob", ctx->emulator_mode ? "yes" : "no", ctx->real_endpoint ? ctx->real_endpoint : "no", - ctx->atype == AZURE_BLOB_AUTH_KEY ? "key" : "sas"); + ctx->atype == AZURE_BLOB_AUTH_KEY ? "key" : + (ctx->atype == AZURE_BLOB_AUTH_SAS ? "sas" : "service_principal")); return ctx; } @@ -840,6 +873,14 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx) flb_sds_destroy(ctx->shared_key_prefix); } + /* Cleanup service principal resources */ + if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL) { + if (ctx->o) { + flb_oauth2_destroy(ctx->o); + ctx->o = NULL; + } + } + if (ctx->u) { flb_upstream_destroy(ctx->u); } diff --git a/plugins/out_azure_blob/azure_blob_http.c b/plugins/out_azure_blob/azure_blob_http.c index 17ca0958570..0483b756582 100644 --- a/plugins/out_azure_blob/azure_blob_http.c +++ b/plugins/out_azure_blob/azure_blob_http.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "azure_blob.h" #include "azure_blob_uri.h" @@ -38,6 +39,111 @@ static int hmac_sha256_sign(unsigned char out[32], out, 32); } +static int azb_get_service_principal_token(struct flb_azure_blob *ctx) +{ + int ret; + char *token; + + flb_oauth2_payload_clear(ctx->o); + + ret = flb_oauth2_payload_append(ctx->o, + "grant_type", 10, + "client_credentials", 18); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to append OAuth2 grant type"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, + "scope", 5, + AZURE_BLOB_OAUTH_SCOPE, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to append OAuth2 scope"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, + "client_id", 9, + ctx->client_id, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to append OAuth2 client ID"); + return -1; + } + + ret = flb_oauth2_payload_append(ctx->o, + "client_secret", 13, + ctx->client_secret, -1); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to append OAuth2 client secret"); + return -1; + } + + token = flb_oauth2_token_get(ctx->o); + if (!token) { + flb_plg_error(ctx->ins, "failed to retrieve OAuth2 access token"); + return -1; + } + + return 0; +} + +/* + * Get the current Azure Blob bearer token as a formatted string. + * Acquires the token mutex, refreshes the token if expired, and returns + * a copy of the token string in the format " ". + * The caller must destroy the returned flb_sds_t. + */ +static int azb_get_oauth2_authorization_header(struct flb_azure_blob *ctx, + flb_sds_t *out_auth_header) +{ + int ret; + flb_sds_t auth_header = NULL; + + if (!out_auth_header) { + return -1; + } + + /* Acquire token mutex */ + ret = pthread_mutex_lock(&ctx->token_mutex); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to lock token mutex"); + return -1; + } + + /* Check if token needs refresh */ + if (flb_oauth2_token_expired(ctx->o) == FLB_TRUE) { + ret = azb_get_service_principal_token(ctx); + if (ret != 0) { + pthread_mutex_unlock(&ctx->token_mutex); + flb_plg_error(ctx->ins, "failed to refresh Service Principal OAuth2 token"); + return -1; + } + } + + /* Build Authorization header while holding lock */ + auth_header = flb_sds_create_size(flb_sds_len(ctx->o->token_type) + + flb_sds_len(ctx->o->access_token) + 2); + if (!auth_header) { + pthread_mutex_unlock(&ctx->token_mutex); + flb_plg_error(ctx->ins, "failed to allocate authorization header"); + return -1; + } + + ret = flb_sds_snprintf(&auth_header, flb_sds_alloc(auth_header), + "%s %s", ctx->o->token_type, ctx->o->access_token); + if (ret < 0) { + flb_sds_destroy(auth_header); + pthread_mutex_unlock(&ctx->token_mutex); + flb_plg_error(ctx->ins, "failed to format authorization header"); + return -1; + } + + pthread_mutex_unlock(&ctx->token_mutex); + + *out_auth_header = auth_header; + return 0; +} + static flb_sds_t canonical_headers(struct flb_http_client *c) { flb_sds_t ch; @@ -300,6 +406,7 @@ int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, ssize_t content_length, int blob_type, int content_type, int content_encoding) { + int ret; int len; time_t now; struct tm tm; @@ -358,6 +465,7 @@ int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, /* Azure header: x-ms-version */ flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); + if (ctx->atype == AZURE_BLOB_AUTH_KEY) { can_req = azb_http_canonical_request(ctx, c, content_length, content_type, content_encoding); @@ -374,6 +482,20 @@ int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, flb_sds_destroy(can_req); flb_sds_destroy(auth); } + else if (ctx->atype == AZURE_BLOB_AUTH_SERVICE_PRINCIPAL) { + /* Get OAuth2 authorization header (thread-safe) */ + ret = azb_get_oauth2_authorization_header(ctx, &auth); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to get OAuth2 authorization header"); + return -1; + } + + /* Azure header: authorization */ + flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); + + /* Release buffer */ + flb_sds_destroy(auth); + } /* Set callback context to the HTTP client context */ flb_http_set_callback_context(c, ctx->ins->callback);