From 044a2a8f597fc29e92da5430ba5e8d6f47316036 Mon Sep 17 00:00:00 2001
From: Takahiro Yamashita <nokute78@gmail.com>
Date: Tue, 23 Nov 2021 21:08:04 +0900
Subject: [PATCH 1/5] tests: runtime: out_es: add test case for long
 index(#4311)

Signed-off-by: Takahiro Yamashita <nokute78@gmail.com>
---
 tests/runtime/out_elasticsearch.c | 72 +++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/tests/runtime/out_elasticsearch.c b/tests/runtime/out_elasticsearch.c
index 3b0e0ad1ca6..9ba0c15bcd6 100644
--- a/tests/runtime/out_elasticsearch.c
+++ b/tests/runtime/out_elasticsearch.c
@@ -420,8 +420,80 @@ void flb_test_div0()
     flb_destroy(ctx);
 }
 
+
+static void cb_check_long_index(void *ctx, int ffd,
+                                int res_ret, void *res_data, size_t res_size,
+                                void *data)
+{
+    char *p;
+    char *out_js = res_data;
+    char long_index[256] = {0};
+    int i;
+
+    for (i=0; i<sizeof(long_index)-1; i++) {
+        long_index[i] = '0' + (i%10);
+    }
+
+    p = strstr(out_js, &long_index[0]);
+    TEST_CHECK(p != NULL);
+    flb_free(res_data);
+}
+
+/* https://github.com/fluent/fluent-bit/issues/4311 */
+void flb_test_long_index()
+{
+    int ret;
+    int size = sizeof(JSON_ES) -1;
+    char long_index[256] = {0};
+    flb_ctx_t *ctx;
+    int in_ffd;
+    int out_ffd;
+    int i;
+
+    for (i=0; i<sizeof(long_index)-1; i++) {
+        long_index[i] = '0' + (i%10);
+    }
+
+    /* Create context, flush every second (some checks omitted here) */
+    ctx = flb_create();
+    flb_service_set(ctx, "flush", "1", "grace", "1", NULL);
+
+    /* Lib input mode */
+    in_ffd = flb_input(ctx, (char *) "lib", NULL);
+    flb_input_set(ctx, in_ffd, "tag", "test", NULL);
+
+    /* Elasticsearch output */
+    out_ffd = flb_output(ctx, (char *) "es", NULL);
+    flb_output_set(ctx, out_ffd,
+                   "match", "test",
+                   "generate_id", "true",
+                   "index", &long_index[0],
+                   NULL);
+
+    /* Override defaults of index and type */
+    flb_output_set(ctx, out_ffd,
+                   NULL);
+
+    /* Enable test mode */
+    ret = flb_output_set_test(ctx, out_ffd, "formatter",
+                              cb_check_long_index,
+                              NULL, NULL);
+
+    /* Start */
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+
+    /* Ingest data sample */
+    flb_lib_push(ctx, in_ffd, (char *)JSON_ES, size);
+
+    sleep(2);
+    flb_stop(ctx);
+    flb_destroy(ctx);
+}
+
 /* Test list */
 TEST_LIST = {
+    {"long_index"           , flb_test_long_index },
     {"div0_error"           , flb_test_div0 },
     {"index_type"           , flb_test_index_type },
     {"logstash_format"      , flb_test_logstash_format },

From 6e9fc8e4058ccb06603eef2dd55c9f99ebb027e6 Mon Sep 17 00:00:00 2001
From: Takahiro Yamashita <nokute78@gmail.com>
Date: Tue, 23 Nov 2021 21:08:55 +0900
Subject: [PATCH 2/5] sds: add flb_sds_snprintf_realloc

The function is a wrapper of snprintf.
It can increase sds if sds doesn't have enough buffer.

Signed-off-by: Takahiro Yamashita <nokute78@gmail.com>
---
 include/fluent-bit/flb_sds.h |  1 +
 src/flb_sds.c                | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/include/fluent-bit/flb_sds.h b/include/fluent-bit/flb_sds.h
index 793d46dba38..d5de15ade35 100644
--- a/include/fluent-bit/flb_sds.h
+++ b/include/fluent-bit/flb_sds.h
@@ -108,5 +108,6 @@ flb_sds_t flb_sds_increase(flb_sds_t s, size_t len);
 flb_sds_t flb_sds_copy(flb_sds_t s, const char *str, int len);
 void flb_sds_destroy(flb_sds_t s);
 flb_sds_t flb_sds_printf(flb_sds_t *sds, const char *fmt, ...);
+int flb_sds_snprintf_realloc(flb_sds_t *str, size_t size, const char *fmt, ...);
 
 #endif
diff --git a/src/flb_sds.c b/src/flb_sds.c
index 5ca1e72b815..30f5158545f 100644
--- a/src/flb_sds.c
+++ b/src/flb_sds.c
@@ -407,3 +407,31 @@ void flb_sds_destroy(flb_sds_t s)
     head = FLB_SDS_HEADER(s);
     flb_free(head);
 }
+
+/*
+ * flb_sds_snprintf_realloc is a wrapper of snprintf.
+ * The difference is that this function can increase the buffer of flb_sds_t.
+ */
+int flb_sds_snprintf_realloc(flb_sds_t *str, size_t size, const char *fmt, ...)
+{
+    va_list va;
+    flb_sds_t tmp;
+    int ret;
+
+ retry_snprintf:
+    va_start(va, fmt);
+    ret = vsnprintf(*str, size, fmt, va);
+    if (ret > size) {
+        tmp = flb_sds_increase(*str, ret-size);
+        if (tmp == NULL) {
+            return -1;
+        }
+        *str = tmp;
+        size = ret;
+        va_end(va);
+        goto retry_snprintf;
+    }
+    va_end(va);
+
+    return ret;
+}

From 32d98bd8623da5deb8c06187f2de1b97eb117239 Mon Sep 17 00:00:00 2001
From: Takahiro Yamashita <nokute78@gmail.com>
Date: Tue, 23 Nov 2021 21:11:37 +0900
Subject: [PATCH 3/5] out_es: use snprintf wrapper function for bulk header.

We used fixed char array to construct bulk header.
If long index comes, it needs larger buffer.

out_es uses snprintf wrapper function.
If the buffer is small, the api increases the buffer.

Signed-off-by: Takahiro Yamashita <nokute78@gmail.com>
---
 plugins/out_es/es.c | 83 ++++++++++++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 34 deletions(-)

diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c
index 26606b94e37..5019086dbac 100644
--- a/plugins/out_es/es.c
+++ b/plugins/out_es/es.c
@@ -265,7 +265,7 @@ static int elasticsearch_format(struct flb_config *config,
     msgpack_object root;
     msgpack_object map;
     msgpack_object *obj;
-    char j_index[ES_BULK_HEADER];
+    flb_sds_t j_index;
     struct es_bulk *bulk;
     struct tm tm;
     struct flb_time tms;
@@ -275,6 +275,12 @@ static int elasticsearch_format(struct flb_config *config,
     int es_index_custom_len;
     struct flb_elasticsearch *ctx = plugin_context;
 
+    j_index = flb_sds_create_size(ES_BULK_HEADER);
+    if (j_index == NULL) {
+        flb_errno();
+        return -1;
+    }
+
     /* Iterate the original buffer and perform adjustments */
     msgpack_unpacked_init(&result);
 
@@ -282,6 +288,7 @@ static int elasticsearch_format(struct flb_config *config,
     ret = msgpack_unpack_next(&result, data, bytes, &off);
     if (ret != MSGPACK_UNPACK_SUCCESS) {
         msgpack_unpacked_destroy(&result);
+        flb_sds_destroy(j_index);
         return -1;
     }
 
@@ -292,17 +299,22 @@ static int elasticsearch_format(struct flb_config *config,
          * doing, we just duplicate the content in a new buffer and cleanup.
          */
         msgpack_unpacked_destroy(&result);
+        flb_sds_destroy(j_index);
         return -1;
     }
 
     root = result.data;
     if (root.via.array.size == 0) {
+        msgpack_unpacked_destroy(&result);
+        flb_sds_destroy(j_index);
         return -1;
     }
 
     /* Create the bulk composer */
     bulk = es_bulk_create(bytes);
     if (!bulk) {
+        msgpack_unpacked_destroy(&result);
+        flb_sds_destroy(j_index);
         return -1;
     }
 
@@ -331,16 +343,16 @@ static int elasticsearch_format(struct flb_config *config,
                  ctx->index, &tm);
         es_index = index_formatted;
         if (ctx->suppress_type_name) {
-            index_len = snprintf(j_index,
-                                 ES_BULK_HEADER,
-                                 ES_BULK_INDEX_FMT_WITHOUT_TYPE,
-                                 es_index);
+            index_len = flb_sds_snprintf_realloc(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT_WITHOUT_TYPE,
+                                                 es_index);
         }
         else {
-            index_len = snprintf(j_index,
-                                 ES_BULK_HEADER,
-                                 ES_BULK_INDEX_FMT,
-                                 es_index, ctx->type);
+            index_len = flb_sds_snprintf_realloc(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT,
+                                                 es_index, ctx->type);
         }
     }
 
@@ -443,16 +455,16 @@ static int elasticsearch_format(struct flb_config *config,
             es_index = logstash_index;
             if (ctx->generate_id == FLB_FALSE) {
                 if (ctx->suppress_type_name) {
-                    index_len = snprintf(j_index,
-                                         ES_BULK_HEADER,
-                                         ES_BULK_INDEX_FMT_WITHOUT_TYPE,
-                                         es_index);
+                    index_len = flb_sds_snprintf_realloc(&j_index,
+                                                         flb_sds_alloc(j_index),
+                                                         ES_BULK_INDEX_FMT_WITHOUT_TYPE,
+                                                         es_index);
                 }
                 else {
-                    index_len = snprintf(j_index,
-                                         ES_BULK_HEADER,
-                                         ES_BULK_INDEX_FMT,
-                                         es_index, ctx->type);
+                    index_len = flb_sds_snprintf_realloc(&j_index,
+                                                         flb_sds_alloc(j_index),
+                                                         ES_BULK_INDEX_FMT,
+                                                         es_index, ctx->type);
                 }
             }
         }
@@ -483,6 +495,7 @@ static int elasticsearch_format(struct flb_config *config,
             msgpack_unpacked_destroy(&result);
             msgpack_sbuffer_destroy(&tmp_sbuf);
             es_bulk_destroy(bulk);
+            flb_sds_destroy(j_index);
             return -1;
         }
 
@@ -493,32 +506,32 @@ static int elasticsearch_format(struct flb_config *config,
                      hash[0], hash[1], hash[2], hash[3],
                      hash[4], hash[5], hash[6], hash[7]);
             if (ctx->suppress_type_name) {
-                index_len = snprintf(j_index,
-                                     ES_BULK_HEADER,
-                                     ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
-                                     es_index,  es_uuid);
+                index_len = flb_sds_snprintf_realloc(&j_index,
+                                                     flb_sds_alloc(j_index),
+                                                     ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
+                                                     es_index,  es_uuid);
             }
             else {
-                index_len = snprintf(j_index,
-                                     ES_BULK_HEADER,
-                                     ES_BULK_INDEX_FMT_ID,
-                                     es_index, ctx->type, es_uuid);
+                index_len = flb_sds_snprintf_realloc(&j_index,
+                                                     flb_sds_alloc(j_index),
+                                                     ES_BULK_INDEX_FMT_ID,
+                                                     es_index, ctx->type, es_uuid);
             }
         }
         if (ctx->ra_id_key) {
             id_key_str = es_get_id_value(ctx ,&map);
             if (id_key_str) {
                 if (ctx->suppress_type_name) {
-                    index_len = snprintf(j_index,
-                                         ES_BULK_HEADER,
-                                         ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
-                                         es_index,  id_key_str);
+                    index_len = flb_sds_snprintf_realloc(&j_index,
+                                                         flb_sds_alloc(j_index),
+                                                         ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
+                                                         es_index,  id_key_str);
                 }
                 else {
-                    index_len = snprintf(j_index,
-                                         ES_BULK_HEADER,
-                                         ES_BULK_INDEX_FMT_ID,
-                                         es_index, ctx->type, id_key_str);
+                    index_len = flb_sds_snprintf_realloc(&j_index,
+                                                         flb_sds_alloc(j_index),
+                                                         ES_BULK_INDEX_FMT_ID,
+                                                         es_index, ctx->type, id_key_str);
                 }
                 flb_sds_destroy(id_key_str);
                 id_key_str = NULL;
@@ -531,6 +544,7 @@ static int elasticsearch_format(struct flb_config *config,
         if (!out_buf) {
             msgpack_unpacked_destroy(&result);
             es_bulk_destroy(bulk);
+            flb_sds_destroy(j_index);
             return -1;
         }
 
@@ -544,6 +558,7 @@ static int elasticsearch_format(struct flb_config *config,
             msgpack_unpacked_destroy(&result);
             *out_size = 0;
             es_bulk_destroy(bulk);
+            flb_sds_destroy(j_index);
             return -1;
         }
     }
@@ -563,7 +578,7 @@ static int elasticsearch_format(struct flb_config *config,
         fwrite(*out_data, 1, *out_size, stdout);
         fflush(stdout);
     }
-
+    flb_sds_destroy(j_index);
     return 0;
 }
 

From cfb042d49cff0bb18e15afe217fb03b859940273 Mon Sep 17 00:00:00 2001
From: Takahiro Yamashita <nokute78@gmail.com>
Date: Sat, 18 Dec 2021 08:46:35 +0900
Subject: [PATCH 4/5] sds: rename flb_sds_snprintf_realloc -> flb_sds_snprintf

Signed-off-by: Takahiro Yamashita <nokute78@gmail.com>
---
 include/fluent-bit/flb_sds.h | 2 +-
 src/flb_sds.c                | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/fluent-bit/flb_sds.h b/include/fluent-bit/flb_sds.h
index d5de15ade35..5665aff6612 100644
--- a/include/fluent-bit/flb_sds.h
+++ b/include/fluent-bit/flb_sds.h
@@ -108,6 +108,6 @@ flb_sds_t flb_sds_increase(flb_sds_t s, size_t len);
 flb_sds_t flb_sds_copy(flb_sds_t s, const char *str, int len);
 void flb_sds_destroy(flb_sds_t s);
 flb_sds_t flb_sds_printf(flb_sds_t *sds, const char *fmt, ...);
-int flb_sds_snprintf_realloc(flb_sds_t *str, size_t size, const char *fmt, ...);
+int flb_sds_snprintf(flb_sds_t *str, size_t size, const char *fmt, ...);
 
 #endif
diff --git a/src/flb_sds.c b/src/flb_sds.c
index 30f5158545f..fb99a63a6ef 100644
--- a/src/flb_sds.c
+++ b/src/flb_sds.c
@@ -409,10 +409,10 @@ void flb_sds_destroy(flb_sds_t s)
 }
 
 /*
- * flb_sds_snprintf_realloc is a wrapper of snprintf.
+ * flb_sds_snprintf is a wrapper of snprintf.
  * The difference is that this function can increase the buffer of flb_sds_t.
  */
-int flb_sds_snprintf_realloc(flb_sds_t *str, size_t size, const char *fmt, ...)
+int flb_sds_snprintf(flb_sds_t *str, size_t size, const char *fmt, ...)
 {
     va_list va;
     flb_sds_t tmp;

From 8dbd5c1f7bea271de7361cae2a6fe39098c92f72 Mon Sep 17 00:00:00 2001
From: Takahiro Yamashita <nokute78@gmail.com>
Date: Sat, 18 Dec 2021 09:44:21 +0900
Subject: [PATCH 5/5] out_es: rename flb_sds_snprintf_realloc ->
 flb_sds_snprintf

Signed-off-by: Takahiro Yamashita <nokute78@gmail.com>
---
 plugins/out_es/es.c | 64 ++++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c
index 5019086dbac..50af298e5f8 100644
--- a/plugins/out_es/es.c
+++ b/plugins/out_es/es.c
@@ -343,16 +343,16 @@ static int elasticsearch_format(struct flb_config *config,
                  ctx->index, &tm);
         es_index = index_formatted;
         if (ctx->suppress_type_name) {
-            index_len = flb_sds_snprintf_realloc(&j_index,
-                                                 flb_sds_alloc(j_index),
-                                                 ES_BULK_INDEX_FMT_WITHOUT_TYPE,
-                                                 es_index);
+            index_len = flb_sds_snprintf(&j_index,
+                                         flb_sds_alloc(j_index),
+                                         ES_BULK_INDEX_FMT_WITHOUT_TYPE,
+                                         es_index);
         }
         else {
-            index_len = flb_sds_snprintf_realloc(&j_index,
-                                                 flb_sds_alloc(j_index),
-                                                 ES_BULK_INDEX_FMT,
-                                                 es_index, ctx->type);
+            index_len = flb_sds_snprintf(&j_index,
+                                         flb_sds_alloc(j_index),
+                                         ES_BULK_INDEX_FMT,
+                                         es_index, ctx->type);
         }
     }
 
@@ -455,16 +455,16 @@ static int elasticsearch_format(struct flb_config *config,
             es_index = logstash_index;
             if (ctx->generate_id == FLB_FALSE) {
                 if (ctx->suppress_type_name) {
-                    index_len = flb_sds_snprintf_realloc(&j_index,
-                                                         flb_sds_alloc(j_index),
-                                                         ES_BULK_INDEX_FMT_WITHOUT_TYPE,
-                                                         es_index);
+                    index_len = flb_sds_snprintf(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT_WITHOUT_TYPE,
+                                                 es_index);
                 }
                 else {
-                    index_len = flb_sds_snprintf_realloc(&j_index,
-                                                         flb_sds_alloc(j_index),
-                                                         ES_BULK_INDEX_FMT,
-                                                         es_index, ctx->type);
+                    index_len = flb_sds_snprintf(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT,
+                                                 es_index, ctx->type);
                 }
             }
         }
@@ -506,32 +506,32 @@ static int elasticsearch_format(struct flb_config *config,
                      hash[0], hash[1], hash[2], hash[3],
                      hash[4], hash[5], hash[6], hash[7]);
             if (ctx->suppress_type_name) {
-                index_len = flb_sds_snprintf_realloc(&j_index,
-                                                     flb_sds_alloc(j_index),
-                                                     ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
-                                                     es_index,  es_uuid);
+                index_len = flb_sds_snprintf(&j_index,
+                                             flb_sds_alloc(j_index),
+                                             ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
+                                             es_index,  es_uuid);
             }
             else {
-                index_len = flb_sds_snprintf_realloc(&j_index,
-                                                     flb_sds_alloc(j_index),
-                                                     ES_BULK_INDEX_FMT_ID,
-                                                     es_index, ctx->type, es_uuid);
+                index_len = flb_sds_snprintf(&j_index,
+                                             flb_sds_alloc(j_index),
+                                             ES_BULK_INDEX_FMT_ID,
+                                             es_index, ctx->type, es_uuid);
             }
         }
         if (ctx->ra_id_key) {
             id_key_str = es_get_id_value(ctx ,&map);
             if (id_key_str) {
                 if (ctx->suppress_type_name) {
-                    index_len = flb_sds_snprintf_realloc(&j_index,
-                                                         flb_sds_alloc(j_index),
-                                                         ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
-                                                         es_index,  id_key_str);
+                    index_len = flb_sds_snprintf(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT_ID_WITHOUT_TYPE,
+                                                 es_index,  id_key_str);
                 }
                 else {
-                    index_len = flb_sds_snprintf_realloc(&j_index,
-                                                         flb_sds_alloc(j_index),
-                                                         ES_BULK_INDEX_FMT_ID,
-                                                         es_index, ctx->type, id_key_str);
+                    index_len = flb_sds_snprintf(&j_index,
+                                                 flb_sds_alloc(j_index),
+                                                 ES_BULK_INDEX_FMT_ID,
+                                                 es_index, ctx->type, id_key_str);
                 }
                 flb_sds_destroy(id_key_str);
                 id_key_str = NULL;
