Skip to content

Commit 9664f44

Browse files
nberleenberlee
committed
out_azure_blob: add path templating support comparable to s3
fixes #7515 Signed-off-by: Nico Berlee <nico.berlee@on2it.net>
1 parent ae14454 commit 9664f44

File tree

11 files changed

+834
-47
lines changed

11 files changed

+834
-47
lines changed

‎plugins/out_azure_blob/azure_blob.c‎

Lines changed: 478 additions & 13 deletions
Large diffs are not rendered by default.

‎plugins/out_azure_blob/azure_blob.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <fluent-bit/flb_upstream.h>
2525
#include <fluent-bit/flb_sds.h>
2626
#include <fluent-bit/flb_sqldb.h>
27+
#include <fluent-bit/flb_time.h>
2728

2829
/* Content-Type */
2930
#define AZURE_BLOB_CT "Content-Type"
@@ -62,6 +63,7 @@ struct flb_azure_blob {
6263
flb_sds_t shared_key;
6364
flb_sds_t endpoint;
6465
flb_sds_t path;
66+
int path_templating_enabled;
6567
flb_sds_t date_key;
6668
flb_sds_t auth_type;
6769
flb_sds_t sas_token;
@@ -166,4 +168,10 @@ struct flb_azure_blob {
166168
struct flb_config *config;
167169
};
168170

171+
int azb_resolve_path(struct flb_azure_blob *ctx,
172+
const char *tag,
173+
int tag_len,
174+
const struct flb_time *timestamp,
175+
flb_sds_t *out_path);
176+
169177
#endif

‎plugins/out_azure_blob/azure_blob_appendblob.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,22 @@
2424
#include "azure_blob_conf.h"
2525
#include "azure_blob_uri.h"
2626

27-
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag)
27+
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
28+
const char *path_prefix,
29+
char *tag)
2830
{
2931
flb_sds_t uri;
32+
const char *effective_path;
3033

3134
uri = azb_uri_container(ctx);
3235
if (!uri) {
3336
return NULL;
3437
}
3538

36-
if (ctx->path) {
37-
flb_sds_printf(&uri, "/%s/%s?comp=appendblock", ctx->path, tag);
39+
effective_path = (path_prefix && path_prefix[0] != '\0') ? path_prefix : ctx->path;
40+
41+
if (effective_path && effective_path[0] != '\0') {
42+
flb_sds_printf(&uri, "/%s/%s?comp=appendblock", effective_path, tag);
3843
}
3944
else {
4045
flb_sds_printf(&uri, "/%s?comp=appendblock", tag);

‎plugins/out_azure_blob/azure_blob_appendblob.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
#include <fluent-bit/flb_output_plugin.h>
2424
#include "azure_blob.h"
2525

26-
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag);
26+
flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx,
27+
const char *path_prefix,
28+
char *tag);
2729

2830
#endif

‎plugins/out_azure_blob/azure_blob_blockblob.c‎

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,23 @@
3131
#include "azure_blob_uri.h"
3232
#include "azure_blob_http.h"
3333

34-
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
34+
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
35+
const char *path_prefix,
36+
char *name)
3537
{
3638
flb_sds_t uri;
39+
const char *effective_path;
3740

3841
uri = azb_uri_container(ctx);
3942
if (!uri) {
4043
return NULL;
4144
}
4245

43-
if (ctx->path) {
46+
effective_path = (path_prefix && path_prefix[0] != '\0') ? path_prefix : ctx->path;
47+
48+
if (effective_path && effective_path[0] != '\0') {
4449
flb_sds_printf(&uri, "/%s/%s?comp=blocklist",
45-
ctx->path, name);
50+
effective_path, name);
4651
}
4752
else {
4853
flb_sds_printf(&uri, "/%s?comp=blocklist", name);
@@ -55,13 +60,18 @@ flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name)
5560
return uri;
5661
}
5762

58-
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
59-
char *blockid, uint64_t ms, char *random_str)
63+
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
64+
const char *path_prefix,
65+
char *name,
66+
char *blockid,
67+
uint64_t ms,
68+
char *random_str)
6069
{
6170
int len;
6271
flb_sds_t uri;
6372
char *ext;
6473
char *encoded_blockid;
74+
const char *effective_path;
6575

6676
len = strlen(blockid);
6777
encoded_blockid = azb_uri_encode(blockid, len);
@@ -82,14 +92,16 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
8292
ext = "";
8393
}
8494

85-
if (ctx->path) {
95+
effective_path = (path_prefix && path_prefix[0] != '\0') ? path_prefix : ctx->path;
96+
97+
if (effective_path && effective_path[0] != '\0') {
8698
if (ms > 0) {
8799
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?blockid=%s&comp=block",
88-
ctx->path, name, random_str, ms, ext, encoded_blockid);
100+
effective_path, name, random_str, ms, ext, encoded_blockid);
89101
}
90102
else {
91103
flb_sds_printf(&uri, "/%s/%s.%s%s?blockid=%s&comp=block",
92-
ctx->path, name, random_str, ext, encoded_blockid);
104+
effective_path, name, random_str, ext, encoded_blockid);
93105
}
94106
}
95107
else {
@@ -112,10 +124,14 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name,
112124
}
113125

114126
flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
115-
char *tag, uint64_t ms, char *str)
127+
const char *path_prefix,
128+
char *tag,
129+
uint64_t ms,
130+
char *str)
116131
{
117132
char *ext;
118133
flb_sds_t uri;
134+
const char *effective_path;
119135

120136
uri = azb_uri_container(ctx);
121137
if (!uri) {
@@ -129,9 +145,13 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx,
129145
ext = "";
130146
}
131147

132-
if (ctx->path) {
133-
flb_sds_printf(&uri, "/%s/%s.%s.%" PRIu64 "%s?comp=blocklist", ctx->path, tag, str,
134-
ms, ext);
148+
effective_path = (path_prefix && path_prefix[0] != '\0') ? path_prefix : ctx->path;
149+
150+
if (effective_path && effective_path[0] != '\0') {
151+
flb_sds_printf(&uri,
152+
"/%s/%s.%s.%" PRIu64 "%s?comp=blocklist",
153+
effective_path, tag, str,
154+
ms, ext);
135155
}
136156
else {
137157
flb_sds_printf(&uri, "/%s.%s.%" PRIu64 "%s?comp=blocklist", tag, str, ms, ext);
@@ -331,14 +351,19 @@ int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb
331351
}
332352

333353
/* Commit a single block */
334-
int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str)
354+
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
355+
const char *path_prefix,
356+
char *blockid,
357+
char *tag,
358+
uint64_t ms,
359+
char *str)
335360
{
336361
int ret;
337362
flb_sds_t uri = NULL;
338363
flb_sds_t payload;
339364

340365
/* Compose commit URI */
341-
uri = azb_block_blob_uri_commit(ctx, tag, ms, str);
366+
uri = azb_block_blob_uri_commit(ctx, path_prefix, tag, ms, str);
342367
if (!uri) {
343368
return FLB_ERROR;
344369
}
@@ -419,7 +444,7 @@ int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_i
419444
cfl_sds_cat_safe(&payload, "</BlockList>", 12);
420445
flb_utils_split_free(list);
421446

422-
uri = azb_block_blob_blocklist_uri(ctx, path);
447+
uri = azb_block_blob_blocklist_uri(ctx, NULL, path);
423448
if (!uri) {
424449
flb_sds_destroy(payload);
425450
return -1;

‎plugins/out_azure_blob/azure_blob_blockblob.h‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,24 @@
2323
#include <fluent-bit/flb_output_plugin.h>
2424
#include "azure_blob.h"
2525

26-
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name);
27-
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, char *blockid,
28-
uint64_t ms, char *random_str);
26+
flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx,
27+
const char *path_prefix,
28+
char *name);
29+
flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx,
30+
const char *path_prefix,
31+
char *tag,
32+
char *blockid,
33+
uint64_t ms,
34+
char *random_str);
2935
char *azb_block_blob_id_logs(uint64_t *ms);
3036
char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, uint64_t part_id);
3137

32-
int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms, char *str);
38+
int azb_block_blob_commit_block(struct flb_azure_blob *ctx,
39+
const char *path_prefix,
40+
char *blockid,
41+
char *tag,
42+
uint64_t ms,
43+
char *str);
3344
int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id,
3445
cfl_sds_t path, cfl_sds_t part_ids);
3546

‎plugins/out_azure_blob/azure_blob_conf.c‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <fluent-bit/flb_output_plugin.h>
2121
#include <fluent-bit/flb_base64.h>
2222
#include <fluent-bit/flb_pack.h>
23+
#include <fluent-bit/flb_record_accessor.h>
2324

2425
#include "azure_blob.h"
2526
#include "azure_blob_conf.h"
@@ -582,6 +583,21 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
582583
}
583584
}
584585

586+
/* Sanitize path and mark templating enabled after remote overrides */
587+
if (ctx->path) {
588+
size_t path_len;
589+
590+
path_len = flb_sds_len(ctx->path);
591+
if (path_len > 0 && ctx->path[path_len - 1] == '/') {
592+
ctx->path[path_len - 1] = '\0';
593+
path_len--;
594+
}
595+
596+
if (path_len > 0) {
597+
ctx->path_templating_enabled = FLB_TRUE;
598+
}
599+
}
600+
585601
if (!ctx->container_name) {
586602
flb_plg_error(ctx->ins, "'container_name' has not been set");
587603
return NULL;
@@ -755,13 +771,6 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in
755771
flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name);
756772
}
757773

758-
/* Sanitize path: remove any ending slash */
759-
if (ctx->path) {
760-
if (ctx->path[flb_sds_len(ctx->path) - 1] == '/') {
761-
ctx->path[flb_sds_len(ctx->path) - 1] = '\0';
762-
}
763-
}
764-
765774
/* database file for blob signal handling */
766775
if (ctx->database_file) {
767776
ctx->db = azb_db_open(ctx, ctx->database_file);
@@ -805,6 +814,7 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx)
805814
flb_sds_destroy(ctx->path);
806815
ctx->path = NULL;
807816
}
817+
ctx->path_templating_enabled = FLB_FALSE;
808818

809819
if (ctx->decoded_sk) {
810820
flb_free(ctx->decoded_sk);

‎plugins/out_azure_blob/azure_blob_uri.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,17 +134,22 @@ flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx)
134134
return uri;
135135
}
136136

137-
flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag)
137+
flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx,
138+
const char *path_prefix,
139+
char *tag)
138140
{
139141
flb_sds_t uri;
142+
const char *effective_path;
140143

141144
uri = azb_uri_container(ctx);
142145
if (!uri) {
143146
return NULL;
144147
}
145148

146-
if (ctx->path) {
147-
flb_sds_printf(&uri, "/%s/%s", ctx->path, tag);
149+
effective_path = (path_prefix && path_prefix[0] != '\0') ? path_prefix : ctx->path;
150+
151+
if (effective_path && effective_path[0] != '\0') {
152+
flb_sds_printf(&uri, "/%s/%s", effective_path, tag);
148153
}
149154
else {
150155
flb_sds_printf(&uri, "/%s", tag);

‎plugins/out_azure_blob/azure_blob_uri.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727

2828
flb_sds_t azb_uri_container(struct flb_azure_blob *ctx);
2929
flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx);
30-
flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag);
30+
flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx,
31+
const char *path_prefix,
32+
char *tag);
3133
flb_sds_t azb_uri_encode(const char *uri, size_t len);
3234
flb_sds_t azb_uri_decode(const char *uri, size_t len);
3335

‎tests/internal/CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,13 @@ set(UNIT_TESTS_FILES
5959
storage_dlq.c
6060
)
6161

62+
if(FLB_OUT_AZURE_BLOB)
63+
set(UNIT_TESTS_FILES
64+
${UNIT_TESTS_FILES}
65+
azure_blob_path.c
66+
)
67+
endif()
68+
6269
# TLS helpers
6370
if(FLB_TLS)
6471
set(UNIT_TESTS_FILES
@@ -217,6 +224,10 @@ function(prepare_unit_tests TEST_PREFIX SOURCEFILES)
217224
target_link_libraries(${source_file_we} flb-aws)
218225
endif()
219226

227+
if(FLB_OUT_AZURE_BLOB AND "${source_file_we}" STREQUAL "flb-it-azure_blob_path")
228+
target_link_libraries(${source_file_we} flb-plugin-out_azure_blob)
229+
endif()
230+
220231
if(FLB_STREAM_PROCESSOR)
221232
target_link_libraries(${source_file_we} flb-sp)
222233
endif()

0 commit comments

Comments
 (0)