From 556dfa59e394c677d04da1f75644666cba711698 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:22:04 -0600 Subject: [PATCH 01/42] http_server: unify core listener API and built-in server Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_http_common.h | 5 + include/fluent-bit/http_server/flb_hs.h | 60 +- include/fluent-bit/http_server/flb_hs_utils.h | 11 + .../fluent-bit/http_server/flb_http_server.h | 118 +++- .../http_server/flb_http_server_config_map.h | 38 ++ src/flb_http_common.c | 86 ++- src/http_server/CMakeLists.txt | 1 + src/http_server/api/v1/health.c | 217 +------ src/http_server/api/v1/health.h | 42 +- src/http_server/api/v1/metrics.c | 232 ++----- src/http_server/api/v1/plugins.c | 22 +- src/http_server/api/v1/storage.c | 172 +----- src/http_server/api/v1/trace.c | 91 +-- src/http_server/api/v1/uptime.c | 22 +- src/http_server/api/v2/metrics.c | 204 ++----- src/http_server/api/v2/reload.c | 70 ++- src/http_server/flb_hs.c | 315 ++++++++-- src/http_server/flb_hs_utils.c | 74 ++- src/http_server/flb_http_server.c | 577 +++++++++++++++++- src/http_server/flb_http_server_config_map.c | 98 +++ src/http_server/flb_http_server_http1.c | 53 +- src/http_server/flb_http_server_http2.c | 36 +- 22 files changed, 1618 insertions(+), 926 deletions(-) create mode 100644 include/fluent-bit/http_server/flb_http_server_config_map.h create mode 100644 src/http_server/flb_http_server_config_map.c diff --git a/include/fluent-bit/flb_http_common.h b/include/fluent-bit/flb_http_common.h index f7f0d7df90d..65b70b30751 100644 --- a/include/fluent-bit/flb_http_common.h +++ b/include/fluent-bit/flb_http_common.h @@ -141,6 +141,7 @@ struct flb_http_stream { }; struct flb_aws_provider; +struct flb_connection; /* HTTP REQUEST */ @@ -155,6 +156,8 @@ int flb_http_request_commit(struct flb_http_request *request); char *flb_http_request_get_header(struct flb_http_request *request, char *name); +const char *flb_http_request_get_remote_address(struct flb_http_request *request); + int flb_http_request_set_header(struct flb_http_request *request, char *name, size_t name_length, char *value, size_t value_length); @@ -196,6 +199,8 @@ int flb_http_request_set_body(struct flb_http_request *request, unsigned char *body, size_t body_length, char *compression_algorithm); +int flb_http_request_normalize(struct flb_http_request *request); + int flb_http_request_set_authorization(struct flb_http_request *request, int type, ...); diff --git a/include/fluent-bit/http_server/flb_hs.h b/include/fluent-bit/http_server/flb_hs.h index 1ef031cbffc..f066f1195dc 100644 --- a/include/fluent-bit/http_server/flb_hs.h +++ b/include/fluent-bit/http_server/flb_hs.h @@ -22,8 +22,10 @@ #include #include +#include #include -#include +#include +#include /* * HTTP buffers that contains certain cached data to be used @@ -37,16 +39,52 @@ struct flb_hs_buf { struct mk_list _head; }; +struct flb_health_check_metrics_counter { + int error_limit; + int error_counter; + int retry_failure_limit; + int retry_failure_counter; + int period_limit; + int period_counter; +}; + +struct flb_hs_hc_buf { + int users; + int error_count; + int retry_failure_count; + struct mk_list _head; +}; + +enum flb_hs_route_match_type { + FLB_HS_ROUTE_EXACT = 0, + FLB_HS_ROUTE_PREFIX = 1 +}; + +struct flb_hs; + +typedef int (*flb_hs_endpoint_callback)( + struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response); + +struct flb_hs_route { + const char *path; + int match_type; + flb_hs_endpoint_callback callback; + struct mk_list _head; +}; + struct flb_hs { - mk_ctx_t *ctx; /* Monkey HTTP Context */ - int vid; /* Virtual Host ID */ - int qid_metrics; /* Metrics Message Queue ID */ - int qid_metrics_v2; /* Metrics Message Queue ID for /api/v2 */ - int qid_storage; /* Storage Message Queue ID */ - int qid_health; /* health Message Queue ID */ + struct flb_http_server server; + struct flb_net_setup net_setup; + struct flb_config *config; + struct mk_list routes; + struct mk_list health_metrics; + struct flb_health_check_metrics_counter health_counter; - pthread_t tid; /* Server Thread */ - struct flb_config *config; /* Fluent Bit context */ + struct flb_hs_buf metrics; + struct flb_hs_buf metrics_v2; + struct flb_hs_buf storage_metrics; /* end-point: root */ size_t ep_root_size; @@ -62,5 +100,9 @@ int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size); int flb_hs_destroy(struct flb_hs *ctx); int flb_hs_start(struct flb_hs *hs); +int flb_hs_register_endpoint(struct flb_hs *hs, + const char *path, + int match_type, + flb_hs_endpoint_callback callback); #endif diff --git a/include/fluent-bit/http_server/flb_hs_utils.h b/include/fluent-bit/http_server/flb_hs_utils.h index 930818db183..e9d35f1bd10 100644 --- a/include/fluent-bit/http_server/flb_hs_utils.h +++ b/include/fluent-bit/http_server/flb_hs_utils.h @@ -21,8 +21,19 @@ #define FLB_HS_UTIL_H #include +#include int flb_hs_add_content_type_to_req(mk_request_t *request, int type); +int flb_hs_response_set_content_type(struct flb_http_response *response, int type); +int flb_hs_response_set_payload(struct flb_http_response *response, + int status, + int type, + const void *payload, + size_t payload_size); +int flb_hs_response_send_string(struct flb_http_response *response, + int status, + int type, + const char *payload); /* Content-type */ enum content_type { diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index b9b0a259177..d1f25d861b6 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -54,8 +54,60 @@ #define HTTP_SERVER_STOPPED 3 typedef int (*flb_http_server_request_processor_callback)( - struct flb_http_request *request, - struct flb_http_response *response); + struct flb_http_request *request, + struct flb_http_response *response); + +struct flb_http_server; +struct flb_http_server_runtime; + +typedef int (*flb_http_server_worker_callback)(struct flb_http_server *server, + void *data); + +struct flb_input_instance; + +struct flb_http_server_config { + int http2; + size_t buffer_max_size; + size_t buffer_chunk_size; + size_t max_connections; +}; + +struct flb_http_server_options { + int protocol_version; + uint64_t flags; + flb_http_server_request_processor_callback request_callback; + void *user_data; + + char *address; + unsigned short int port; + struct flb_tls *tls_provider; + int networking_flags; + struct flb_net_setup *networking_setup; + struct mk_event_loop *event_loop; + struct flb_config *system_context; + + size_t buffer_max_size; + size_t max_connections; + + /* Total number of worker listeners to spawn. */ + int workers; + + /* + * If true and workers == 1, run the server on the caller supplied + * event loop. Otherwise the server owns the worker thread(s) and the + * corresponding event loop(s). + */ + int use_caller_event_loop; + + /* + * If true, use a shared listener port across internally managed + * workers. This is forced on automatically when workers > 1. + */ + int reuse_port; + + flb_http_server_worker_callback cb_worker_init; + flb_http_server_worker_callback cb_worker_exit; +}; struct flb_http_server { /* Internal */ @@ -69,15 +121,22 @@ struct flb_http_server { struct flb_config *system_context; /* Internal */ - uint64_t flags; - int status; - int protocol_version; - struct flb_downstream *downstream; - struct cfl_list clients; - flb_http_server_request_processor_callback - request_callback; - void *user_data; - size_t buffer_max_size; + uint64_t flags; + int status; + int protocol_version; + struct flb_downstream *downstream; + struct cfl_list clients; + flb_http_server_request_processor_callback request_callback; + void *user_data; + size_t buffer_max_size; + size_t max_connections; + int workers; + int worker_id; + int use_caller_event_loop; + int reuse_port; + flb_http_server_worker_callback cb_worker_init; + flb_http_server_worker_callback cb_worker_exit; + struct flb_http_server_runtime *runtime; }; struct flb_http_server_session { @@ -106,8 +165,7 @@ struct flb_http_server_session { int flb_http_server_init(struct flb_http_server *session, int protocol_version, uint64_t flags, - flb_http_server_request_processor_callback - request_callback, + flb_http_server_request_processor_callback request_callback, char *address, unsigned short int port, struct flb_tls *tls_provider, @@ -117,12 +175,42 @@ int flb_http_server_init(struct flb_http_server *session, struct flb_config *system_context, void *user_data); +void flb_http_server_options_init(struct flb_http_server_options *options); + +void flb_http_server_config_init(struct flb_http_server_config *config); + +int flb_http_server_options_init_from_input(struct flb_http_server_options *options, + struct flb_input_instance *input_instance, + int protocol_version, + uint64_t flags, + size_t buffer_max_size, + flb_http_server_request_processor_callback + request_callback, + void *user_data); + +int flb_input_http_server_options_init(struct flb_http_server_options *options, + struct flb_input_instance *input_instance, + uint64_t flags, + flb_http_server_request_processor_callback request_callback, + void *user_data); + +int flb_http_server_init_with_options(struct flb_http_server *session, + struct flb_http_server_options *options); + int flb_http_server_start(struct flb_http_server *session); int flb_http_server_stop(struct flb_http_server *session); int flb_http_server_destroy(struct flb_http_server *session); +int flb_http_server_init_on_input(struct flb_http_server *session, + struct flb_input_instance *input_instance, + int protocol_version, + uint64_t flags, + size_t buffer_max_size, + flb_http_server_request_processor_callback request_callback, + void *user_data); + void flb_http_server_set_buffer_max_size(struct flb_http_server *server, size_t size); size_t flb_http_server_get_buffer_max_size(struct flb_http_server *server); @@ -136,7 +224,7 @@ struct flb_http_server_session *flb_http_server_session_create(int version); void flb_http_server_session_destroy(struct flb_http_server_session *session); int flb_http_server_session_ingest(struct flb_http_server_session *session, - unsigned char *buffer, - size_t length); + unsigned char *buffer, + size_t length); #endif diff --git a/include/fluent-bit/http_server/flb_http_server_config_map.h b/include/fluent-bit/http_server/flb_http_server_config_map.h new file mode 100644 index 00000000000..ee858c60749 --- /dev/null +++ b/include/fluent-bit/http_server/flb_http_server_config_map.h @@ -0,0 +1,38 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HTTP_SERVER_CONFIG_MAP_H +#define FLB_HTTP_SERVER_CONFIG_MAP_H + +#include + +#include +#include + +extern struct flb_config_map flb_http_server_config_map[]; + +struct mk_list *flb_http_server_get_config_map(struct flb_config *config); + +int flb_http_server_property_is_allowed(const char *property_name); + +int flb_http_server_config_map_set(struct mk_list *properties, + struct mk_list *config_map, + struct flb_http_server_config *context); + +#endif diff --git a/src/flb_http_common.c b/src/flb_http_common.c index 554ea44a852..7449d723ddf 100644 --- a/src/flb_http_common.c +++ b/src/flb_http_common.c @@ -256,6 +256,28 @@ char *flb_http_request_get_header(struct flb_http_request *request, return (char *) value; } +const char *flb_http_request_get_remote_address(struct flb_http_request *request) +{ + struct flb_http_server_session *server_session; + + if (request == NULL || request->stream == NULL) { + return NULL; + } + + if (request->stream->role != HTTP_STREAM_ROLE_SERVER || + request->stream->parent == NULL) { + return NULL; + } + + server_session = (struct flb_http_server_session *) request->stream->parent; + + if (server_session->connection == NULL) { + return NULL; + } + + return flb_connection_get_remote_address(server_session->connection); +} + int flb_http_request_set_header(struct flb_http_request *request, char *name, size_t name_length, char *value, size_t value_length) @@ -781,6 +803,51 @@ int flb_http_request_set_body(struct flb_http_request *request, return 0; } +int flb_http_request_normalize(struct flb_http_request *request) +{ + char *query_separator; + cfl_sds_t normalized_path; + cfl_sds_t normalized_query_string; + cfl_sds_t original_path; + size_t path_length; + + if (request == NULL || request->path == NULL) { + return 0; + } + + original_path = request->path; + + query_separator = strchr(original_path, '?'); + + if (query_separator == NULL) { + return 0; + } + + path_length = (size_t) (query_separator - original_path); + + normalized_path = cfl_sds_create_len(original_path, path_length); + if (normalized_path == NULL) { + return -1; + } + + normalized_query_string = cfl_sds_create(&query_separator[1]); + if (normalized_query_string == NULL) { + cfl_sds_destroy(normalized_path); + return -1; + } + + if (request->query_string != NULL) { + cfl_sds_destroy(request->query_string); + } + + request->path = normalized_path; + request->query_string = normalized_query_string; + + cfl_sds_destroy(original_path); + + return 0; +} + int flb_http_request_perform_signv4_signature( struct flb_http_request *request, const char *aws_region, @@ -928,9 +995,27 @@ int flb_http_response_commit(struct flb_http_response *response) int len; char tmp[64]; int version; + char *server_header; version = flb_http_response_get_version(response); + server_header = flb_http_response_get_header(response, "server"); + if (server_header == NULL) { + flb_http_response_set_header(response, + "server", + strlen("server"), + "Fluent Bit", + strlen("Fluent Bit")); + } + + if (flb_http_response_get_header(response, "x-http-engine") == NULL) { + flb_http_response_set_header(response, + "x-http-engine", + strlen("x-http-engine"), + "Monkey heritage", + strlen("Monkey heritage")); + } + if (response->body == NULL) { flb_http_response_set_header(response, "content-length", @@ -1645,4 +1730,3 @@ int compress_gzip(char **output_buffer, return 1; } - diff --git a/src/http_server/CMakeLists.txt b/src/http_server/CMakeLists.txt index 8004475b35f..e9958cae20e 100644 --- a/src/http_server/CMakeLists.txt +++ b/src/http_server/CMakeLists.txt @@ -8,6 +8,7 @@ set(src flb_hs_endpoints.c flb_hs_utils.c flb_http_server.c + flb_http_server_config_map.c flb_http_server_http1.c flb_http_server_http2.c ) diff --git a/src/http_server/api/v1/health.c b/src/http_server/api/v1/health.c index 94aa90a92fd..f3ee3abc2b9 100644 --- a/src/http_server/api/v1/health.c +++ b/src/http_server/api/v1/health.c @@ -25,69 +25,11 @@ #include #include #include +#include #include #include "health.h" -struct flb_health_check_metrics_counter *metrics_counter; - -pthread_key_t hs_health_key; - -static struct mk_list *hs_health_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_health_key, metrics_list); - - return metrics_list; -} - -static void hs_health_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_hc_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_hc_buf, _head); - if (entry != NULL) { - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - -/* initialize the metrics counters */ -static void counter_init(struct flb_hs *hs) { - - metrics_counter = flb_malloc(sizeof(struct flb_health_check_metrics_counter)); - - if (!metrics_counter) { - flb_errno(); - return; - } - - metrics_counter->error_counter = 0; - metrics_counter->retry_failure_counter = 0; - metrics_counter->error_limit = hs->config->hc_errors_count; - metrics_counter->retry_failure_limit = hs->config->hc_retry_failure_count; - metrics_counter->period_counter = 0; - metrics_counter->period_limit = hs->config->health_check_period; - -} - /* * tell what's the current status for health check * One default background is that the metrics received and saved into @@ -97,27 +39,18 @@ static void counter_init(struct flb_hs *hs) { * the error number of the newest metrics message minus * the error number in oldest metrics of period */ -static int is_healthy() { - - struct mk_list *metrics_list; +static int is_healthy(struct flb_hs *hs) +{ struct flb_hs_hc_buf *buf; int period_errors; int period_retry_failure; - metrics_list = pthread_getspecific(hs_health_key); - if (metrics_list == NULL) { - metrics_list = hs_health_key_create(); - if (metrics_list == NULL) { - return FLB_FALSE; - } - } - - if (mk_list_is_empty(metrics_list) == 0) { + if (mk_list_is_empty(&hs->health_metrics) == 0) { return FLB_TRUE; } /* Get the error metrics entry from the start time of current period */ - buf = mk_list_entry_first(metrics_list, struct flb_hs_hc_buf, _head); + buf = mk_list_entry_first(&hs->health_metrics, struct flb_hs_hc_buf, _head); /* * increase user so clean up function won't @@ -130,13 +63,13 @@ static int is_healthy() { * the error count in current period = (current error count in total) - * (begin error count in the period) */ - period_errors = metrics_counter->error_counter - buf->error_count; - period_retry_failure = metrics_counter->retry_failure_counter - + period_errors = hs->health_counter.error_counter - buf->error_count; + period_retry_failure = hs->health_counter.retry_failure_counter - buf->retry_failure_count; buf->users--; - if (period_errors > metrics_counter->error_limit || - period_retry_failure > metrics_counter->retry_failure_limit) { + if (period_errors > hs->health_counter.error_limit || + period_retry_failure > hs->health_counter.retry_failure_limit) { return FLB_FALSE; } @@ -145,8 +78,8 @@ static int is_healthy() { } /* read the metrics from message queue and update the counter*/ -static void read_metrics(void *data, size_t size, int* error_count, - int* retry_failure_count) +void read_metrics(void *data, size_t size, int* error_count, + int* retry_failure_count) { int i; int j; @@ -205,131 +138,37 @@ static void read_metrics(void *data, size_t size, int* error_count, msgpack_unpacked_destroy(&result); } -/* -* Delete unused metrics, note that we only care about the latest node -* we use this function to maintain the metrics queue only save the metrics -* in a period. The old metrics which is out of period will be removed -*/ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_hc_buf *entry; - - metrics_list = pthread_getspecific(hs_health_key); - if (!metrics_list) { - return -1; - } - - if (metrics_counter->period_counter < metrics_counter->period_limit) { - return 0; - } - - /* remove the oldest metrics if it's out of period */ - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_hc_buf, _head); - if (metrics_counter->period_counter > metrics_counter->period_limit && - entry->users == 0) { - metrics_counter->period_counter--; - mk_list_del(&entry->_head); - flb_free(entry); - c++; - } - else { - break; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and record the health - * status based on the metrics. - * This happens every second based on the event config. - * So we treat period_counter to count the time. - * And we maintain a message queue with the size of period limit number - * so every time we get a new metrics data in, if the message queue size is - * large than period limit, we will do the clean up func to - * remove the oldest metrics. - */ -static void cb_mq_health(mk_mq_t *queue, void *data, size_t size) -{ - struct flb_hs_hc_buf *buf; - struct mk_list *metrics_list = NULL; - int error_count = 0; - int retry_failure_count = 0; - - metrics_list = pthread_getspecific(hs_health_key); - - if (metrics_list == NULL) { - metrics_list = hs_health_key_create(); - if (metrics_list == NULL) { - return; - } - } - - metrics_counter->period_counter++; - - /* this is to remove the metrics out of period*/ - cleanup_metrics(); - - buf = flb_malloc(sizeof(struct flb_hs_hc_buf)); - if (!buf) { - flb_errno(); - return; - } - - buf->users = 0; - - read_metrics(data, size, &error_count, &retry_failure_count); - - metrics_counter->error_counter = error_count; - metrics_counter->retry_failure_counter = retry_failure_count; - - buf->error_count = error_count; - buf->retry_failure_count = retry_failure_count; - - mk_list_add(&buf->_head, metrics_list); -} - /* API: Get fluent Bit Health Status */ -static void cb_health(mk_request_t *request, void *data) +static int cb_health(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { - int status = is_healthy(); + int ret; + int status = is_healthy(hs); + + (void) hs; + (void) request; if (status == FLB_TRUE) { - mk_http_status(request, 200); - mk_http_send(request, "ok\n", strlen("ok\n"), NULL); - mk_http_done(request); + ret = flb_hs_response_send_string(response, 200, + FLB_HS_CONTENT_TYPE_OTHER, "ok\n"); } else { - mk_http_status(request, 500); - mk_http_send(request, "error\n", strlen("error\n"), NULL); - mk_http_done(request); + ret = flb_hs_response_send_string(response, 500, + FLB_HS_CONTENT_TYPE_OTHER, "error\n"); } + + return ret; } /* Perform registration */ int api_v1_health(struct flb_hs *hs) { - - pthread_key_create(&hs_health_key, hs_health_key_destroy); - - counter_init(hs); - /* Create a message queue */ - hs->qid_health = mk_mq_create(hs->ctx, "/health", - cb_mq_health, NULL); - - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/health", cb_health, hs); - return 0; + return flb_hs_register_endpoint(hs, "/api/v1/health", + FLB_HS_ROUTE_EXACT, cb_health); } void flb_hs_health_destroy() { - flb_free(metrics_counter); + return; } diff --git a/src/http_server/api/v1/health.h b/src/http_server/api/v1/health.h index 1d2e5097ad9..a0a4415831b 100644 --- a/src/http_server/api/v1/health.h +++ b/src/http_server/api/v1/health.h @@ -25,48 +25,10 @@ #include #include -struct flb_health_check_metrics_counter { - - /* - * health check error limit, - * setup by customer through config: HC_Errors_Count - */ - int error_limit; - - /* counter the error number in metrics*/ - int error_counter; - - /* - * health check retry failed limit, - * setup by customer through config: HC_Retry_Failure_Count - */ - int retry_failure_limit; - - /* count the retry failed number in metrics*/ - int retry_failure_counter; - - /*period limit, setup by customer through config: HC_Period*/ - int period_limit; - - /* count the seconds in one period*/ - int period_counter; - -}; - - -/* - * error and retry failure buffers that contains certain cached data to be used - * by health check. - */ -struct flb_hs_hc_buf { - int users; - int error_count; - int retry_failure_count; - struct mk_list _head; -}; - /* health endpoint*/ int api_v1_health(struct flb_hs *hs); +void read_metrics(void *data, size_t size, int *error_count, + int *retry_failure_count); /* clean up health resource when shutdown*/ void flb_hs_health_destroy(); diff --git a/src/http_server/api/v1/metrics.c b/src/http_server/api/v1/metrics.c index 5421d0f4a11..991c84a610a 100644 --- a/src/http_server/api/v1/metrics.c +++ b/src/http_server/api/v1/metrics.c @@ -28,158 +28,18 @@ #include "metrics.h" #include +#include #include #define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0) -pthread_key_t hs_metrics_key; - -static struct mk_list *hs_metrics_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (metrics_list == NULL) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_metrics_key, metrics_list); - - return metrics_list; -} - -static void hs_metrics_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - flb_free(entry->raw_data); - entry->raw_data = NULL; - } - if (entry->data) { - flb_sds_destroy(entry->data); - entry->data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - /* Return the newest metrics buffer */ -static struct flb_hs_buf *metrics_get_latest() +static struct flb_hs_buf *metrics_get_latest(struct flb_hs *hs) { - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { + if (hs->metrics.data == NULL || hs->metrics.raw_data == NULL) { return NULL; } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { - return -1; - } - - last = metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_sds_destroy(entry->data); - flb_free(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and store it - * somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - flb_sds_t out_data; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_metrics_key); - if (!metrics_list) { - metrics_list = hs_metrics_key_create(); - if (metrics_list == NULL) { - return; - } - } - - /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); - if (!out_data) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - flb_sds_destroy(out_data); - return; - } - buf->users = 0; - buf->data = out_data; - - buf->raw_data = flb_malloc(size); - if (!buf->raw_data) { - flb_errno(); - flb_sds_destroy(out_data); - flb_free(buf); - return; - } - memcpy(buf->raw_data, data, size); - buf->raw_size = size; - - mk_list_add(&buf->_head, metrics_list); - - cleanup_metrics(); + return &hs->metrics; } int string_cmp(const void* a_arg, const void* b_arg) { @@ -260,7 +120,9 @@ flb_sds_t metrics_help_txt(char *metric_name, flb_sds_t *metric_helptxt) } /* API: expose metrics in Prometheus format /api/v1/metrics/prometheus */ -void cb_metrics_prometheus(mk_request_t *request, void *data) +static int cb_metrics_prometheus(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { int i; int j; @@ -286,14 +148,14 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) char start_time_str[64]; char* *metrics_arr; struct flb_time tp; - struct flb_hs *hs = data; struct flb_config *config = hs->config; - buf = metrics_get_latest(); + (void) request; + + buf = metrics_get_latest(hs); if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } /* ref count */ @@ -302,20 +164,20 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) /* Compose outgoing buffer string */ sds = flb_sds_create_size(1024); if (!sds) { - mk_http_status(request, 500); - mk_http_done(request); + flb_http_response_set_status(response, 500); + flb_http_response_commit(response); buf->users--; - return; + return 0; } /* length of HELP text */ metric_helptxt = flb_sds_create_size(128); if (!metric_helptxt) { flb_sds_destroy(sds); - mk_http_status(request, 500); - mk_http_done(request); + flb_http_response_set_status(response, 500); + flb_http_response_commit(response); buf->users--; - return; + return 0; } metric_helptxt_head = FLB_SDS_HEADER(metric_helptxt); @@ -343,14 +205,14 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) if (!metrics_arr) { flb_errno(); - mk_http_status(request, 500); - mk_http_done(request); buf->users--; + flb_http_response_set_status(response, 500); + flb_http_response_commit(response); flb_sds_destroy(sds); flb_sds_destroy(metric_helptxt); msgpack_unpacked_destroy(&result); - return; + return 0; } flb_time_get(&tp); @@ -511,9 +373,9 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) msgpack_unpacked_destroy(&result); buf->users--; - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, sds, flb_sds_len(sds), NULL); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + sds, flb_sds_len(sds)); for (i = 0; i < num_metrics; i++) { flb_sds_destroy(metrics_arr[i]); } @@ -521,12 +383,11 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) flb_sds_destroy(sds); flb_sds_destroy(metric_helptxt); - mk_http_done(request); - return; + return 0; error: - mk_http_status(request, 500); - mk_http_done(request); + flb_http_response_set_status(response, 500); + flb_http_response_commit(response); buf->users--; for (i = 0; i < index; i++) { @@ -536,44 +397,39 @@ void cb_metrics_prometheus(mk_request_t *request, void *data) flb_sds_destroy(sds); flb_sds_destroy(metric_helptxt); msgpack_unpacked_destroy(&result); + return 0; } /* API: expose built-in metrics /api/v1/metrics */ -static void cb_metrics(mk_request_t *request, void *data) +static int cb_metrics(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { struct flb_hs_buf *buf; - buf = metrics_get_latest(); + (void) request; + + buf = metrics_get_latest(hs); if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } buf->users++; - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, buf->data, flb_sds_len(buf->data), NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_JSON, + buf->data, flb_sds_len(buf->data)); buf->users--; + return 0; } /* Perform registration */ int api_v1_metrics(struct flb_hs *hs) { - - pthread_key_create(&hs_metrics_key, hs_metrics_key_destroy); - - /* Create a message queue */ - hs->qid_metrics = mk_mq_create(hs->ctx, "/metrics", - cb_mq_metrics, NULL); - - /* HTTP end-points */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/metrics/prometheus", - cb_metrics_prometheus, hs); - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/metrics", cb_metrics, hs); - - return 0; + flb_hs_register_endpoint(hs, "/api/v1/metrics/prometheus", + FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + return flb_hs_register_endpoint(hs, "/api/v1/metrics", + FLB_HS_ROUTE_EXACT, cb_metrics); } diff --git a/src/http_server/api/v1/plugins.c b/src/http_server/api/v1/plugins.c index b5356187841..760ab45b41c 100644 --- a/src/http_server/api/v1/plugins.c +++ b/src/http_server/api/v1/plugins.c @@ -24,10 +24,13 @@ #include #include #include +#include #include /* API: List all built-in plugins */ -static void cb_plugins(mk_request_t *request, void *data) +static int cb_plugins(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { int len; flb_sds_t out_buf; @@ -37,9 +40,10 @@ static void cb_plugins(mk_request_t *request, void *data) struct flb_input_plugin *in; struct flb_filter_plugin *filter; struct flb_output_plugin *out; - struct flb_hs *hs = data; struct flb_config *config = hs->config; + (void) request; + /* initialize buffers */ msgpack_sbuffer_init(&mp_sbuf); msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); @@ -93,17 +97,19 @@ static void cb_plugins(mk_request_t *request, void *data) out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); - mk_http_status(request, 200); - mk_http_send(request, - out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, + 200, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, + flb_sds_len(out_buf)); flb_sds_destroy(out_buf); + return 0; } /* Perform registration */ int api_v1_plugins(struct flb_hs *hs) { - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/plugins", cb_plugins, hs); - return 0; + return flb_hs_register_endpoint(hs, "/api/v1/plugins", + FLB_HS_ROUTE_EXACT, cb_plugins); } diff --git a/src/http_server/api/v1/storage.c b/src/http_server/api/v1/storage.c index 6ab6d19800c..77f51e842d0 100644 --- a/src/http_server/api/v1/storage.c +++ b/src/http_server/api/v1/storage.c @@ -23,182 +23,46 @@ #include "storage.h" #include +#include #include -pthread_key_t hs_storage_metrics_key; - /* Return the newest storage metrics buffer */ -static struct flb_hs_buf *storage_metrics_get_latest() +static struct flb_hs_buf *storage_metrics_get_latest(struct flb_hs *hs) { - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { + if (hs->storage_metrics.data == NULL) { return NULL; } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { - return -1; - } - - last = storage_metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_sds_destroy(entry->data); - flb_free(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some storage metrics are received through a - * message queue channel. This function runs in a Monkey HTTP thread - * worker and it purpose is to take the metrics data and store it - * somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_storage_metrics(mk_mq_t *queue, void *data, size_t size) -{ - flb_sds_t out_data; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_storage_metrics_key); - if (!metrics_list) { - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_storage_metrics_key, metrics_list); - } - - /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); - if (!out_data) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - flb_sds_destroy(out_data); - return; - } - buf->users = 0; - buf->data = out_data; - - buf->raw_data = flb_malloc(size); - memcpy(buf->raw_data, data, size); - buf->raw_size = size; - - mk_list_add(&buf->_head, metrics_list); - - cleanup_metrics(); + return &hs->storage_metrics; } -/* FIXME: pending implementation of metrics exit interface -static void cb_mq_storage_metrics_exit(mk_mq_t *queue, void *data) -{ - -} -*/ - /* API: expose built-in storage metrics /api/v1/storage */ -static void cb_storage(mk_request_t *request, void *data) +static int cb_storage(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { struct flb_hs_buf *buf; - buf = storage_metrics_get_latest(); + (void) request; + + buf = storage_metrics_get_latest(hs); if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } buf->users++; - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, buf->data, flb_sds_len(buf->data), NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_JSON, + buf->data, flb_sds_len(buf->data)); buf->users--; -} - -static void hs_storage_metrics_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*)data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - flb_free(entry->raw_data); - entry->raw_data = NULL; - } - if (entry->data) { - flb_sds_destroy(entry->data); - entry->data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); + return 0; } /* Perform registration */ int api_v1_storage_metrics(struct flb_hs *hs) { - pthread_key_create(&hs_storage_metrics_key, hs_storage_metrics_key_destroy); - - /* Create a message queue */ - hs->qid_storage = mk_mq_create(hs->ctx, "/storage", - cb_mq_storage_metrics, - NULL); - - /* HTTP end-point */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/storage", cb_storage, hs); - - return 0; + return flb_hs_register_endpoint(hs, "/api/v1/storage", + FLB_HS_ROUTE_EXACT, cb_storage); } diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index 65bfa3e6cf0..c82b8e6ae8f 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #define STR_INPUTS "inputs" @@ -108,23 +109,23 @@ static int disable_trace_input(struct flb_hs *hs, const char *name, size_t nlen) return 201; } -static flb_sds_t get_input_name(mk_request_t *request) +static flb_sds_t get_input_name(struct flb_http_request *request) { const char base[] = "/api/v1/trace/"; - if (request->real_path.data == NULL) { + if (request->path == NULL) { return NULL; } - if (request->real_path.len < sizeof(base)-1) { + if (cfl_sds_len(request->path) < sizeof(base)-1) { return NULL; } - return flb_sds_create_len(&request->real_path.data[sizeof(base)-1], - request->real_path.len - (sizeof(base)-1)); + return flb_sds_create_len(&request->path[sizeof(base)-1], + cfl_sds_len(request->path) - (sizeof(base)-1)); } -static int http_disable_trace(mk_request_t *request, void *data, +static int http_disable_trace(struct flb_http_request *request, void *data, const char *input_name, size_t input_nlen, msgpack_packer *mp_pck) { @@ -240,7 +241,7 @@ static int msgpack_params_enable_trace(struct flb_hs *hs, msgpack_unpacked *resu return ret; } -static int http_enable_trace(mk_request_t *request, void *data, +static int http_enable_trace(struct flb_http_request *request, void *data, const char *input_name, ssize_t input_nlen, msgpack_packer *mp_pck) { @@ -263,7 +264,7 @@ static int http_enable_trace(mk_request_t *request, void *data, struct flb_input_instance *input_instance; - if (request->method == MK_METHOD_GET) { + if (request->method == HTTP_METHOD_GET) { ret = enable_trace_input(hs, input_name, input_nlen, "trace.", "stdout", NULL); if (ret == 0) { msgpack_pack_map(mp_pck, 1); @@ -278,7 +279,7 @@ static int http_enable_trace(mk_request_t *request, void *data, } msgpack_unpacked_init(&result); - rc = flb_pack_json(request->data.data, request->data.len, &buf, &buf_size, + rc = flb_pack_json(request->body, cfl_sds_len(request->body), &buf, &buf_size, &root_type, NULL); if (rc == -1) { ret = 503; @@ -429,12 +430,14 @@ static int http_enable_trace(mk_request_t *request, void *data, return ret; } -static void cb_trace(mk_request_t *request, void *data) +static int cb_trace(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *http_response) { flb_sds_t out_buf; msgpack_sbuffer mp_sbuf; msgpack_packer mp_pck; - int response = 404; + int http_status = 404; flb_sds_t input_name = NULL; @@ -444,23 +447,23 @@ static void cb_trace(mk_request_t *request, void *data) input_name = get_input_name(request); if (input_name == NULL) { - response = 404; + http_status = 404; goto error; } - if (request->method == MK_METHOD_POST || request->method == MK_METHOD_GET) { - response = http_enable_trace(request, data, input_name, flb_sds_len(input_name), &mp_pck); + if (request->method == HTTP_METHOD_POST || request->method == HTTP_METHOD_GET) { + http_status = http_enable_trace(request, hs, input_name, flb_sds_len(input_name), &mp_pck); } - else if (request->method == MK_METHOD_DELETE) { - response = http_disable_trace(request, data, input_name, flb_sds_len(input_name), &mp_pck); + else if (request->method == HTTP_METHOD_DELETE) { + http_status = http_disable_trace(request, hs, input_name, flb_sds_len(input_name), &mp_pck); } error: - if (response == 404) { + if (http_status == 404) { msgpack_pack_map(&mp_pck, 1); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_NOTFOUND, HTTP_RESULT_NOTFOUND_LEN); } - else if (response == 503) { + else if (http_status == 503) { msgpack_pack_map(&mp_pck, 1); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_ERROR, HTTP_RESULT_ERROR_LEN); @@ -473,20 +476,22 @@ static void cb_trace(mk_request_t *request, void *data) /* Export to JSON */ out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); if (out_buf == NULL) { - mk_http_status(request, 503); - mk_http_done(request); - return; + flb_http_response_set_status(http_response, 503); + return flb_http_response_commit(http_response); } - mk_http_status(request, response); - mk_http_send(request, out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); + flb_hs_response_set_payload(http_response, http_status, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, flb_sds_len(out_buf)); msgpack_sbuffer_destroy(&mp_sbuf); flb_sds_destroy(out_buf); + return 0; } -static void cb_traces(mk_request_t *request, void *data) +static int cb_traces(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *http_response) { flb_sds_t out_buf; msgpack_sbuffer mp_sbuf; @@ -497,7 +502,7 @@ static void cb_traces(mk_request_t *request, void *data) int root_type = MSGPACK_OBJECT_ARRAY; msgpack_unpacked result; flb_sds_t error_msg = NULL; - int response = 200; + int http_status = 200; const char *input_name; ssize_t input_nlen; msgpack_object_array *inputs = NULL; @@ -509,7 +514,7 @@ static void cb_traces(mk_request_t *request, void *data) msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); msgpack_unpacked_init(&result); - ret = flb_pack_json(request->data.data, request->data.len, &buf, &buf_size, + ret = flb_pack_json(request->body, cfl_sds_len(request->body), &buf, &buf_size, &root_type, NULL); if (ret == -1) { goto unpack_error; @@ -523,7 +528,7 @@ static void cb_traces(mk_request_t *request, void *data) } if (result.data.type != MSGPACK_OBJECT_MAP) { - response = 503; + http_status = 503; error_msg = flb_sds_create("input is not an object"); goto unpack_error; } @@ -545,7 +550,7 @@ static void cb_traces(mk_request_t *request, void *data) } if (inputs == NULL) { - response = 503; + http_status = 503; error_msg = flb_sds_create("inputs not found"); goto unpack_error; } @@ -558,7 +563,7 @@ static void cb_traces(mk_request_t *request, void *data) for (i = 0; i < inputs->size; i++) { if (inputs->ptr[i].type != MSGPACK_OBJECT_STR || inputs->ptr[i].via.str.ptr == NULL) { - response = 503; + http_status = 503; error_msg = flb_sds_create("invalid input"); msgpack_sbuffer_clear(&mp_sbuf); goto unpack_error; @@ -572,9 +577,9 @@ static void cb_traces(mk_request_t *request, void *data) msgpack_pack_str_with_body(&mp_pck, input_name, input_nlen); - if (request->method == MK_METHOD_POST) { + if (request->method == HTTP_METHOD_POST) { - ret = msgpack_params_enable_trace((struct flb_hs *)data, &result, + ret = msgpack_params_enable_trace(hs, &result, input_name, input_nlen); if (ret != 0) { @@ -591,8 +596,8 @@ static void cb_traces(mk_request_t *request, void *data) msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_OK, HTTP_RESULT_OK_LEN); } } - else if (request->method == MK_METHOD_DELETE) { - disable_trace_input((struct flb_hs *)data, input_name, input_nlen); + else if (request->method == HTTP_METHOD_DELETE) { + disable_trace_input(hs, input_name, input_nlen); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_OK, HTTP_RESULT_OK_LEN); } @@ -612,12 +617,12 @@ static void cb_traces(mk_request_t *request, void *data) flb_free(buf); } msgpack_unpacked_destroy(&result); - if (response == 404) { + if (http_status == 404) { msgpack_pack_map(&mp_pck, 1); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_NOTFOUND, HTTP_RESULT_NOTFOUND_LEN); } - else if (response == 503) { + else if (http_status == 503) { msgpack_pack_map(&mp_pck, 2); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_OK, HTTP_RESULT_OK_LEN); @@ -644,20 +649,22 @@ static void cb_traces(mk_request_t *request, void *data) } msgpack_sbuffer_destroy(&mp_sbuf); - mk_http_status(request, response); - mk_http_send(request, - out_buf, flb_sds_len(out_buf), NULL); - mk_http_done(request); + flb_hs_response_set_payload(http_response, http_status, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, flb_sds_len(out_buf)); flb_sds_destroy(out_buf); + return 0; } /* Perform registration */ int api_v1_trace(struct flb_hs *hs) { if (hs->config->enable_chunk_trace == FLB_TRUE) { - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/traces/", cb_traces, hs); - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/trace/*", cb_trace, hs); + flb_hs_register_endpoint(hs, "/api/v1/traces/", + FLB_HS_ROUTE_EXACT, cb_traces); + flb_hs_register_endpoint(hs, "/api/v1/trace/", + FLB_HS_ROUTE_PREFIX, cb_trace); } return 0; } diff --git a/src/http_server/api/v1/uptime.c b/src/http_server/api/v1/uptime.c index 28f4c3bbc47..d451f9784d7 100644 --- a/src/http_server/api/v1/uptime.c +++ b/src/http_server/api/v1/uptime.c @@ -21,6 +21,7 @@ #include #include #include +#include #define FLB_UPTIME_ONEDAY 86400 #define FLB_UPTIME_ONEHOUR 3600 @@ -64,16 +65,19 @@ static void uptime_hr(time_t uptime, msgpack_packer *mp_pck) } /* API: List all built-in plugins */ -static void cb_uptime(mk_request_t *request, void *data) +static int cb_uptime(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { flb_sds_t out_buf; size_t out_size; time_t uptime; msgpack_packer mp_pck; msgpack_sbuffer mp_sbuf; - struct flb_hs *hs = data; struct flb_config *config = hs->config; + (void) request; + /* initialize buffers */ msgpack_sbuffer_init(&mp_sbuf); msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); @@ -91,21 +95,21 @@ static void cb_uptime(mk_request_t *request, void *data) out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } out_size = flb_sds_len(out_buf); - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, FLB_HS_CONTENT_TYPE_JSON, + out_buf, out_size); flb_sds_destroy(out_buf); + return 0; } /* Perform registration */ int api_v1_uptime(struct flb_hs *hs) { - mk_vhost_handler(hs->ctx, hs->vid, "/api/v1/uptime", cb_uptime, hs); - return 0; + return flb_hs_register_endpoint(hs, "/api/v1/uptime", + FLB_HS_ROUTE_EXACT, cb_uptime); } diff --git a/src/http_server/api/v2/metrics.c b/src/http_server/api/v2/metrics.c index 567bccf3713..cbe02af9a89 100644 --- a/src/http_server/api/v2/metrics.c +++ b/src/http_server/api/v2/metrics.c @@ -28,159 +28,34 @@ #include "metrics.h" #include +#include #define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0) -pthread_key_t hs_metrics_v2_key; - -static struct mk_list *hs_metrics_v2_key_create() -{ - struct mk_list *metrics_list = NULL; - - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (metrics_list == NULL) { - flb_errno(); - return NULL; - } - mk_list_init(metrics_list); - pthread_setspecific(hs_metrics_v2_key, metrics_list); - - return metrics_list; -} - -static void hs_metrics_v2_key_destroy(void *data) -{ - struct mk_list *metrics_list = (struct mk_list*) data; - struct mk_list *tmp; - struct mk_list *head; - struct flb_hs_buf *entry; - - if (metrics_list == NULL) { - return; - } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != NULL) { - if (entry->raw_data != NULL) { - cmt_destroy(entry->raw_data); - entry->raw_data = NULL; - } - mk_list_del(&entry->_head); - flb_free(entry); - } - } - - flb_free(metrics_list); -} - /* Return the newest metrics buffer */ -static struct flb_hs_buf *metrics_get_latest() +static struct flb_hs_buf *metrics_get_latest(struct flb_hs *hs) { - struct flb_hs_buf *buf; - struct mk_list *metrics_list; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { + if (hs->metrics_v2.raw_data == NULL) { return NULL; } - - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct flb_hs_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct flb_hs_buf *last; - struct flb_hs_buf *entry; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { - return -1; - } - - last = metrics_get_latest(); - if (!last) { - return -1; - } - - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct flb_hs_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - cmt_destroy(entry->raw_data); - flb_free(entry); - c++; - } - } - - return c; -} - -/* - * Callback invoked every time some metrics are received through a message queue channel. - * This function runs in a Monkey HTTP thread worker and it purpose is to take the metrics - * data and store it somewhere so then it can be available by the end-points upon - * HTTP client requests. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - int ret; - size_t off = 0; - struct cmt *cmt; - struct flb_hs_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(hs_metrics_v2_key); - if (!metrics_list) { - metrics_list = hs_metrics_v2_key_create(); - if (metrics_list == NULL) { - return; - } - } - - /* decode cmetrics */ - ret = cmt_decode_msgpack_create(&cmt, data, size, &off); - if (ret != 0) { - return; - } - - buf = flb_malloc(sizeof(struct flb_hs_buf)); - if (!buf) { - flb_errno(); - return; - } - buf->users = 0; - buf->data = NULL; - - /* Store CMetrics context as the raw_data */ - buf->raw_data = cmt; - buf->raw_size = 0; - - mk_list_add(&buf->_head, metrics_list); - cleanup_metrics(); + return &hs->metrics_v2; } /* API: expose metrics in Prometheus format /api/v2/metrics/prometheus */ -static void cb_metrics_prometheus(mk_request_t *request, void *data) +static int cb_metrics_prometheus(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { struct cmt *cmt; struct flb_hs_buf *buf; cfl_sds_t payload; - buf = metrics_get_latest(); + (void) request; + + buf = metrics_get_latest(hs); if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } cmt = (struct cmt *) buf->raw_data; @@ -188,35 +63,37 @@ static void cb_metrics_prometheus(mk_request_t *request, void *data) /* convert CMetrics to text */ payload = cmt_encode_prometheus_create(cmt, CMT_FALSE); if (!payload) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } buf->users++; - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, payload, cfl_sds_len(payload), NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + payload, cfl_sds_len(payload)); cmt_encode_prometheus_destroy(payload); buf->users--; + return 0; } /* API: expose built-in metrics /api/v1/metrics (JSON format) */ -static void cb_metrics(mk_request_t *request, void *data) +static int cb_metrics(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { struct cmt *cmt; struct flb_hs_buf *buf; cfl_sds_t payload; - buf = metrics_get_latest(); + (void) request; + + buf = metrics_get_latest(hs); if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } cmt = (struct cmt *) buf->raw_data; @@ -224,36 +101,27 @@ static void cb_metrics(mk_request_t *request, void *data) /* convert CMetrics to text */ payload = cmt_encode_text_create(cmt); if (!payload) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } buf->users++; - mk_http_status(request, 200); - mk_http_send(request, payload, cfl_sds_len(payload), NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_OTHER, + payload, cfl_sds_len(payload)); cmt_encode_text_destroy(payload); buf->users--; + return 0; } /* Perform registration */ int api_v2_metrics(struct flb_hs *hs) { - - pthread_key_create(&hs_metrics_v2_key, hs_metrics_v2_key_destroy); - - /* Create a message queue */ - hs->qid_metrics_v2 = mk_mq_create(hs->ctx, "/metrics_v2", - cb_mq_metrics, NULL); - /* HTTP end-points */ - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics/prometheus", - cb_metrics_prometheus, hs); - - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/metrics", cb_metrics, hs); - - return 0; + flb_hs_register_endpoint(hs, "/api/v2/metrics/prometheus", + FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + return flb_hs_register_endpoint(hs, "/api/v2/metrics", + FLB_HS_ROUTE_EXACT, cb_metrics); } diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index 1f82ab8cb2e..cd8953f35a4 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -27,13 +27,15 @@ #include #include #include +#include #include "reload.h" #include #include -static void handle_reload_request(mk_request_t *request, struct flb_config *config) +static int handle_reload_request(struct flb_http_response *response, + struct flb_config *config) { int ret; flb_sds_t out_buf; @@ -69,9 +71,8 @@ static void handle_reload_request(mk_request_t *request, struct flb_config *conf else { ret = GenerateConsoleCtrlEvent(1 /* CTRL_BREAK_EVENT_1 */, 0); if (ret == 0) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } msgpack_pack_str(&mp_pck, 4); @@ -99,9 +100,8 @@ static void handle_reload_request(mk_request_t *request, struct flb_config *conf else { ret = kill(getpid(), SIGHUP); if (ret != 0) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } msgpack_pack_str(&mp_pck, 4); @@ -117,21 +117,21 @@ static void handle_reload_request(mk_request_t *request, struct flb_config *conf out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { - mk_http_status(request, 400); - mk_http_done(request); - return; + flb_http_response_set_status(response, 400); + return flb_http_response_commit(response); } out_size = flb_sds_len(out_buf); - mk_http_status(request, http_status); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, http_status, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, out_size); flb_sds_destroy(out_buf); + return 0; } -static void handle_get_reload_status(mk_request_t *request, struct flb_config *config) +static int handle_get_reload_status(struct flb_http_response *response, + struct flb_config *config) { flb_sds_t out_buf; size_t out_size; @@ -151,42 +151,40 @@ static void handle_get_reload_status(mk_request_t *request, struct flb_config *c out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { - mk_http_status(request, 400); - mk_http_done(request); - return; + flb_http_response_set_status(response, 400); + return flb_http_response_commit(response); } out_size = flb_sds_len(out_buf); - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, out_buf, out_size, NULL); - mk_http_done(request); + flb_hs_response_set_payload(response, 200, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, out_size); flb_sds_destroy(out_buf); + return 0; } -static void cb_reload(mk_request_t *request, void *data) +static int cb_reload(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) { - struct flb_hs *hs = data; struct flb_config *config = hs->config; - if (request->method == MK_METHOD_POST || - request->method == MK_METHOD_PUT) { - handle_reload_request(request, config); + if (request->method == HTTP_METHOD_POST || + request->method == HTTP_METHOD_PUT) { + return handle_reload_request(response, config); } - else if (request->method == MK_METHOD_GET) { - handle_get_reload_status(request, config); - } - else { - mk_http_status(request, 400); - mk_http_done(request); + else if (request->method == HTTP_METHOD_GET) { + return handle_get_reload_status(response, config); } + + flb_http_response_set_status(response, 400); + return flb_http_response_commit(response); } /* Perform registration */ int api_v2_reload(struct flb_hs *hs) { - mk_vhost_handler(hs->ctx, hs->vid, "/api/v2/reload", cb_reload, hs); - - return 0; + return flb_hs_register_endpoint(hs, "/api/v2/reload", + FLB_HS_ROUTE_EXACT, cb_reload); } diff --git a/src/http_server/flb_hs.c b/src/http_server/flb_hs.c index 5c9ec2c129f..42a6995d407 100644 --- a/src/http_server/flb_hs.c +++ b/src/http_server/flb_hs.c @@ -20,10 +20,11 @@ #include #include #include +#include +#include #include -#include - -#include +#include +#include /* v1 */ #include "api/v1/register.h" @@ -32,79 +33,298 @@ /* v2 */ #include "api/v2/register.h" -static void cb_root(mk_request_t *request, void *data) +static int flb_hs_route_matches(struct flb_hs_route *route, cfl_sds_t path) +{ + size_t route_length; + size_t path_length; + + if (route == NULL || route->path == NULL || path == NULL) { + return FLB_FALSE; + } + + route_length = strlen(route->path); + path_length = cfl_sds_len(path); + + if (route->match_type == FLB_HS_ROUTE_PREFIX) { + if (path_length < route_length) { + return FLB_FALSE; + } + + return strncmp(route->path, path, route_length) == 0; + } + + if (route_length != path_length) { + return FLB_FALSE; + } + + return strncmp(route->path, path, route_length) == 0; +} + +static void flb_hs_destroy_cmt_buffer(void *data) +{ + cmt_destroy((struct cmt *) data); +} + +static int cb_root(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) +{ + (void) request; + + return flb_hs_response_set_payload(response, + 200, + FLB_HS_CONTENT_TYPE_JSON, + hs->ep_root_buf, + hs->ep_root_size); +} + +static int flb_hs_request_handler(struct flb_http_request *request, + struct flb_http_response *response) +{ + struct mk_list *head; + struct flb_hs *hs; + struct flb_hs_route *route; + + hs = response->stream->user_data; + if (hs == NULL) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } + + mk_list_foreach(head, &hs->routes) { + route = mk_list_entry(head, struct flb_hs_route, _head); + + if (flb_hs_route_matches(route, request->path)) { + return route->callback(hs, request, response); + } + } + + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); +} + +static void flb_hs_buf_cleanup(struct flb_hs_buf *buffer, + void (*raw_free)(void *)) { - struct flb_hs *hs = data; + if (buffer == NULL) { + return; + } + + if (buffer->data != NULL) { + flb_sds_destroy(buffer->data); + buffer->data = NULL; + } + + if (buffer->raw_data != NULL) { + if (raw_free != NULL) { + raw_free(buffer->raw_data); + } + else { + flb_free(buffer->raw_data); + } + buffer->raw_data = NULL; + } - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_JSON); - mk_http_send(request, hs->ep_root_buf, hs->ep_root_size, NULL); - mk_http_done(request); + buffer->raw_size = 0; + buffer->users = 0; +} + +int flb_hs_register_endpoint(struct flb_hs *hs, + const char *path, + int match_type, + flb_hs_endpoint_callback callback) +{ + struct flb_hs_route *route; + + route = flb_calloc(1, sizeof(struct flb_hs_route)); + if (route == NULL) { + flb_errno(); + return -1; + } + + route->path = path; + route->match_type = match_type; + route->callback = callback; + + mk_list_add(&route->_head, &hs->routes); + + return 0; } /* Ingest health metrics into the web service context */ int flb_hs_push_health_metrics(struct flb_hs *hs, void *data, size_t size) { - return mk_mq_send(hs->ctx, hs->qid_health, data, size); + struct flb_hs_hc_buf *buf; + int error_count; + int retry_failure_count; + + if (hs == NULL) { + return -1; + } + + read_metrics(data, size, &error_count, &retry_failure_count); + + hs->health_counter.period_counter++; + + while (hs->health_counter.period_counter > hs->health_counter.period_limit && + mk_list_size(&hs->health_metrics) > 0) { + buf = mk_list_entry_first(&hs->health_metrics, struct flb_hs_hc_buf, _head); + mk_list_del(&buf->_head); + flb_free(buf); + hs->health_counter.period_counter--; + } + + buf = flb_calloc(1, sizeof(struct flb_hs_hc_buf)); + if (buf == NULL) { + flb_errno(); + return -1; + } + + buf->error_count = error_count; + buf->retry_failure_count = retry_failure_count; + + hs->health_counter.error_counter = error_count; + hs->health_counter.retry_failure_counter = retry_failure_count; + + mk_list_add(&buf->_head, &hs->health_metrics); + + return 0; } /* Ingest pipeline metrics into the web service context */ int flb_hs_push_pipeline_metrics(struct flb_hs *hs, void *data, size_t size) { - return mk_mq_send(hs->ctx, hs->qid_metrics, data, size); + flb_sds_t json_buffer; + void *raw_buffer; + + if (hs == NULL) { + return -1; + } + + json_buffer = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); + if (json_buffer == NULL) { + return -1; + } + + raw_buffer = flb_malloc(size); + if (raw_buffer == NULL) { + flb_errno(); + flb_sds_destroy(json_buffer); + return -1; + } + + memcpy(raw_buffer, data, size); + + flb_hs_buf_cleanup(&hs->metrics, NULL); + + hs->metrics.data = json_buffer; + hs->metrics.raw_data = raw_buffer; + hs->metrics.raw_size = size; + + return 0; } /* Ingest pipeline metrics into the web service context */ int flb_hs_push_metrics(struct flb_hs *hs, void *data, size_t size) { - return mk_mq_send(hs->ctx, hs->qid_metrics_v2, data, size); + int ret; + size_t off = 0; + struct cmt *cmt; + + if (hs == NULL) { + return -1; + } + + ret = cmt_decode_msgpack_create(&cmt, data, size, &off); + if (ret != 0) { + return -1; + } + + flb_hs_buf_cleanup(&hs->metrics_v2, flb_hs_destroy_cmt_buffer); + + hs->metrics_v2.raw_data = cmt; + + return 0; } /* Ingest storage metrics into the web service context */ int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size) { - return mk_mq_send(hs->ctx, hs->qid_storage, data, size); + flb_sds_t json_buffer; + void *raw_buffer; + + if (hs == NULL) { + return -1; + } + + json_buffer = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); + if (json_buffer == NULL) { + return -1; + } + + raw_buffer = flb_malloc(size); + if (raw_buffer == NULL) { + flb_errno(); + flb_sds_destroy(json_buffer); + return -1; + } + + memcpy(raw_buffer, data, size); + + flb_hs_buf_cleanup(&hs->storage_metrics, NULL); + + hs->storage_metrics.data = json_buffer; + hs->storage_metrics.raw_data = raw_buffer; + hs->storage_metrics.raw_size = size; + + return 0; } /* Create ROOT endpoints */ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, struct flb_config *config) { - int vid; - /* Accept IPv6 and IPv4 address */ - char tmp[46]; + int ret; struct flb_hs *hs; + struct flb_http_server_options options; hs = flb_calloc(1, sizeof(struct flb_hs)); if (!hs) { flb_errno(); return NULL; } + hs->config = config; + mk_list_init(&hs->routes); + mk_list_init(&hs->health_metrics); + + hs->health_counter.error_limit = config->hc_errors_count; + hs->health_counter.retry_failure_limit = config->hc_retry_failure_count; + hs->health_counter.period_limit = config->health_check_period; /* Setup endpoint specific data */ flb_hs_endpoints(hs); - /* Create HTTP server context */ - hs->ctx = mk_create(); - if (!hs->ctx) { - flb_error("[http_server] could not create context"); + flb_http_server_options_init(&options); + options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + options.flags = 0; + options.request_callback = flb_hs_request_handler; + options.user_data = hs; + options.address = (char *) listen; + options.port = atoi(tcp_port); + options.networking_flags = 0; + flb_net_setup_init(&hs->net_setup); + options.networking_setup = &hs->net_setup; + options.event_loop = config->evl; + options.system_context = config; + options.use_caller_event_loop = FLB_TRUE; + + ret = flb_http_server_init_with_options(&hs->server, &options); + if (ret != 0) { flb_free(hs); return NULL; } - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%s", listen, tcp_port); - mk_config_set(hs->ctx, "Listen", tmp, NULL); - vid = mk_vhost_create(hs->ctx, NULL); - hs->vid = vid; - - /* Setup virtual host */ - mk_vhost_set(hs->ctx, vid, - "Name", "fluent-bit", - NULL); - - /* Register endpoints for /api/v1 */ api_v1_registration(hs); @@ -112,7 +332,7 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, api_v2_registration(hs); /* Root */ - mk_vhost_handler(hs->ctx, vid, "/", cb_root, hs); + flb_hs_register_endpoint(hs, "/", FLB_HS_ROUTE_EXACT, cb_root); return hs; } @@ -122,27 +342,48 @@ int flb_hs_start(struct flb_hs *hs) int ret; struct flb_config *config = hs->config; - ret = mk_start(hs->ctx); + ret = flb_http_server_start(&hs->server); if (ret == 0) { flb_info("[http_server] listen iface=%s tcp_port=%s", config->http_listen, config->http_port); } + return ret; } int flb_hs_destroy(struct flb_hs *hs) { + struct mk_list *head; + struct mk_list *tmp; + struct flb_hs_route *route; + struct flb_hs_hc_buf *health_buffer; + if (!hs) { return 0; } + flb_hs_health_destroy(); - mk_stop(hs->ctx); - mk_destroy(hs->ctx); + flb_http_server_destroy(&hs->server); + + flb_hs_buf_cleanup(&hs->metrics, NULL); + flb_hs_buf_cleanup(&hs->metrics_v2, flb_hs_destroy_cmt_buffer); + flb_hs_buf_cleanup(&hs->storage_metrics, NULL); + + mk_list_foreach_safe(head, tmp, &hs->health_metrics) { + health_buffer = mk_list_entry(head, struct flb_hs_hc_buf, _head); + mk_list_del(&health_buffer->_head); + flb_free(health_buffer); + } + + mk_list_foreach_safe(head, tmp, &hs->routes) { + route = mk_list_entry(head, struct flb_hs_route, _head); + mk_list_del(&route->_head); + flb_free(route); + } flb_hs_endpoints_free(hs); flb_free(hs); - return 0; } diff --git a/src/http_server/flb_hs_utils.c b/src/http_server/flb_hs_utils.c index 487db9ff540..dc78ad3ab51 100644 --- a/src/http_server/flb_hs_utils.c +++ b/src/http_server/flb_hs_utils.c @@ -17,10 +17,9 @@ * limitations under the License. */ - #include -#include -#include +#include +#include int flb_hs_add_content_type_to_req(mk_request_t *request, int type) { @@ -39,6 +38,8 @@ int flb_hs_add_content_type_to_req(mk_request_t *request, int type) FLB_HS_CONTENT_TYPE_KEY_STR, FLB_HS_CONTENT_TYPE_KEY_LEN, FLB_HS_CONTENT_TYPE_PROMETHEUS_STR, FLB_HS_CONTENT_TYPE_PROMETHEUS_LEN); break; + case FLB_HS_CONTENT_TYPE_OTHER: + break; default: flb_error("[%s] unknown type=%d", __FUNCTION__, type); return -1; @@ -46,3 +47,70 @@ int flb_hs_add_content_type_to_req(mk_request_t *request, int type) return 0; } + +int flb_hs_response_set_content_type(struct flb_http_response *response, int type) +{ + if (response == NULL) { + return -1; + } + + switch (type) { + case FLB_HS_CONTENT_TYPE_JSON: + flb_http_response_set_header(response, + FLB_HS_CONTENT_TYPE_KEY_STR, FLB_HS_CONTENT_TYPE_KEY_LEN, + FLB_HS_CONTENT_TYPE_JSON_STR, FLB_HS_CONTENT_TYPE_JSON_LEN); + break; + case FLB_HS_CONTENT_TYPE_PROMETHEUS: + flb_http_response_set_header(response, + FLB_HS_CONTENT_TYPE_KEY_STR, FLB_HS_CONTENT_TYPE_KEY_LEN, + FLB_HS_CONTENT_TYPE_PROMETHEUS_STR, FLB_HS_CONTENT_TYPE_PROMETHEUS_LEN); + break; + case FLB_HS_CONTENT_TYPE_OTHER: + break; + default: + flb_error("[%s] unknown type=%d", __FUNCTION__, type); + return -1; + } + + return 0; +} + +int flb_hs_response_set_payload(struct flb_http_response *response, + int status, + int type, + const void *payload, + size_t payload_size) +{ + if (response == NULL) { + return -1; + } + + flb_http_response_set_status(response, status); + flb_hs_response_set_content_type(response, type); + + if (payload != NULL && payload_size > 0) { + flb_http_response_set_body(response, (unsigned char *) payload, payload_size); + } + else { + flb_http_response_set_body(response, NULL, 0); + } + + return flb_http_response_commit(response); +} + +int flb_hs_response_send_string(struct flb_http_response *response, + int status, + int type, + const char *payload) +{ + size_t payload_size; + + if (payload == NULL) { + payload_size = 0; + } + else { + payload_size = strlen(payload); + } + + return flb_hs_response_set_payload(response, status, type, payload, payload_size); +} diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 9271eadecaf..cc811d8c4cd 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -18,15 +18,112 @@ */ #include +#include +#include +#include #include +#include #include +#include #include #include /* PRIVATE */ +struct flb_http_server_worker_context { + struct flb_http_server parent; + struct flb_http_server server; + struct flb_net_setup net_setup; + struct mk_event_loop *event_loop; + pthread_t thread; + pthread_mutex_t mutex; + pthread_cond_t condition; + struct mk_list downstreams; + int worker_id; + int should_exit; + int initialized; + int startup_result; +}; + +struct flb_http_server_runtime { + struct flb_http_server_worker_context *workers; + int worker_count; +}; + +static void flb_http_server_runtime_stop(struct flb_http_server *session); + +static void flb_http_server_worker_context_reset( + struct flb_http_server_worker_context *worker) +{ + memset(worker, 0, sizeof(struct flb_http_server_worker_context)); + mk_list_init(&worker->downstreams); + pthread_mutex_init(&worker->mutex, NULL); + pthread_cond_init(&worker->condition, NULL); +} + +static void flb_http_server_worker_context_cleanup( + struct flb_http_server_worker_context *worker) +{ + pthread_mutex_destroy(&worker->mutex); + pthread_cond_destroy(&worker->condition); +} + +static int flb_http_server_running_on_caller_context( + struct flb_http_server *session) +{ + return session->workers == 1 && + session->use_caller_event_loop == FLB_TRUE && + session->event_loop != NULL; +} + +static size_t flb_http_server_client_count(struct flb_http_server *server) +{ + return cfl_list_size(&server->clients); +} + +static int flb_http_server_apply_options(struct flb_http_server *session, + struct flb_http_server_options *options) +{ + if (session == NULL || options == NULL) { + return -1; + } + + session->status = HTTP_SERVER_UNINITIALIZED; + session->protocol_version = options->protocol_version; + session->flags = options->flags; + session->request_callback = options->request_callback; + session->user_data = options->user_data; + + session->address = options->address; + session->port = options->port; + session->tls_provider = options->tls_provider; + session->networking_flags = options->networking_flags; + session->networking_setup = options->networking_setup; + session->event_loop = options->event_loop; + session->system_context = options->system_context; + + session->downstream = NULL; + session->buffer_max_size = options->buffer_max_size; + session->max_connections = options->max_connections; + session->workers = options->workers; + session->worker_id = 0; + session->use_caller_event_loop = options->use_caller_event_loop; + session->reuse_port = options->reuse_port; + session->cb_worker_init = options->cb_worker_init; + session->cb_worker_exit = options->cb_worker_exit; + session->runtime = NULL; + + cfl_list_init(&session->clients); + + MK_EVENT_NEW(&session->listener_event); + + session->status = HTTP_SERVER_INITIALIZED; + + return 0; +} + static int flb_http_server_session_read(struct flb_http_server_session *session) { size_t sent; @@ -49,7 +146,10 @@ static int flb_http_server_session_read(struct flb_http_server_session *session) result); if (result == HTTP_SERVER_BUFFER_LIMIT_EXCEEDED) { - flb_io_net_write(session->connection, (void *) request_too_large, strlen(request_too_large), &sent); + flb_io_net_write(session->connection, + (void *) request_too_large, + strlen(request_too_large), + &sent); return -1; } else if (result < 0) { @@ -264,6 +364,13 @@ static int flb_http_server_client_connection_event_handler(void *data) return -1; } + if (server->max_connections > 0 && + flb_http_server_client_count(server) >= server->max_connections) { + flb_downstream_conn_release(connection); + + return -5; + } + session = flb_http_server_session_create(server->protocol_version); if (session == NULL) { @@ -275,6 +382,10 @@ static int flb_http_server_client_connection_event_handler(void *data) session->parent = server; session->connection = connection; + if (session->version <= HTTP_PROTOCOL_VERSION_11) { + session->http1.stream.user_data = server->user_data; + } + MK_EVENT_NEW(&connection->event); connection->user_data = (void *) session; @@ -306,13 +417,223 @@ static int flb_http_server_client_connection_event_handler(void *data) return 0; } +static void flb_http_server_worker_maintenance(struct flb_config *config, + void *data) +{ + struct flb_http_server_worker_context *worker; + + (void) config; + + worker = data; + + flb_downstream_conn_timeouts(&worker->downstreams); +} + +static int flb_http_server_worker_initialize( + struct flb_http_server_worker_context *worker) +{ + int result; + struct flb_http_server_options options; + + flb_http_server_options_init(&options); + + options.protocol_version = worker->parent.protocol_version; + options.flags = worker->parent.flags; + options.request_callback = worker->parent.request_callback; + options.user_data = worker->parent.user_data; + options.address = worker->parent.address; + options.port = worker->parent.port; + options.tls_provider = worker->parent.tls_provider; + options.networking_flags = worker->parent.networking_flags; + options.networking_setup = &worker->net_setup; + options.event_loop = worker->event_loop; + options.system_context = worker->parent.system_context; + options.buffer_max_size = worker->parent.buffer_max_size; + options.workers = 1; + options.use_caller_event_loop = FLB_TRUE; + options.reuse_port = worker->parent.reuse_port; + options.cb_worker_init = worker->parent.cb_worker_init; + options.cb_worker_exit = worker->parent.cb_worker_exit; + + result = flb_http_server_init_with_options(&worker->server, &options); + if (result != 0) { + return result; + } + + result = flb_http_server_start(&worker->server); + if (result != 0) { + return result; + } + + worker->server.worker_id = worker->worker_id; + worker->server.workers = worker->parent.workers; + + mk_list_add(&worker->server.downstream->base._head, &worker->downstreams); + + return 0; +} + +static void *flb_http_server_worker_thread(void *data) +{ + int result; + struct mk_event *event; + struct flb_net_dns dns_ctx = {0}; + struct flb_http_server_worker_context *worker; + + worker = data; + + worker->event_loop = mk_event_loop_create(256); + if (worker->event_loop == NULL) { + result = -1; + goto signal_and_exit; + } + + flb_engine_evl_set(worker->event_loop); + flb_net_ctx_init(&dns_ctx); + flb_net_dns_ctx_set(&dns_ctx); + + result = flb_http_server_worker_initialize(worker); + +signal_and_exit: + pthread_mutex_lock(&worker->mutex); + worker->startup_result = result; + worker->initialized = FLB_TRUE; + pthread_cond_signal(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + + if (result != 0) { + goto cleanup; + } + + while (worker->should_exit == FLB_FALSE) { + mk_event_wait_2(worker->event_loop, 250); + + mk_event_foreach(event, worker->event_loop) { + if (event->type == FLB_ENGINE_EV_CUSTOM) { + event->handler(event); + } + } + + flb_http_server_worker_maintenance(worker->parent.system_context, + worker); + flb_downstream_conn_pending_destroy_list(&worker->downstreams); + } + +cleanup: + flb_http_server_destroy(&worker->server); + + if (worker->event_loop != NULL) { + mk_event_loop_destroy(worker->event_loop); + worker->event_loop = NULL; + } + + return NULL; +} + +static int __attribute__((unused)) +flb_http_server_runtime_start(struct flb_http_server *session) +{ + int index; + int result; + struct flb_http_server_runtime *runtime; + + runtime = flb_calloc(1, sizeof(struct flb_http_server_runtime)); + if (runtime == NULL) { + flb_errno(); + return -1; + } + + runtime->workers = flb_calloc(session->workers, + sizeof(struct flb_http_server_worker_context)); + if (runtime->workers == NULL) { + flb_errno(); + flb_free(runtime); + return -1; + } + + runtime->worker_count = session->workers; + session->runtime = runtime; + + for (index = 0; index < runtime->worker_count; index++) { + flb_http_server_worker_context_reset(&runtime->workers[index]); + memcpy(&runtime->workers[index].parent, + session, + sizeof(struct flb_http_server)); + memcpy(&runtime->workers[index].net_setup, + session->networking_setup, + sizeof(struct flb_net_setup)); + + runtime->workers[index].net_setup.share_port = FLB_TRUE; + runtime->workers[index].worker_id = index; + runtime->workers[index].parent.reuse_port = FLB_TRUE; + runtime->workers[index].parent.runtime = NULL; + runtime->workers[index].parent.workers = session->workers; + + result = pthread_create(&runtime->workers[index].thread, + NULL, + flb_http_server_worker_thread, + &runtime->workers[index]); + if (result != 0) { + runtime->workers[index].startup_result = -1; + runtime->workers[index].initialized = FLB_TRUE; + break; + } + + pthread_mutex_lock(&runtime->workers[index].mutex); + while (runtime->workers[index].initialized == FLB_FALSE) { + pthread_cond_wait(&runtime->workers[index].condition, + &runtime->workers[index].mutex); + } + result = runtime->workers[index].startup_result; + pthread_mutex_unlock(&runtime->workers[index].mutex); + + if (result != 0) { + break; + } + } + + if (index != runtime->worker_count) { + flb_http_server_runtime_stop(session); + return -1; + } + + session->status = HTTP_SERVER_RUNNING; + + return 0; +} + +static void flb_http_server_runtime_stop(struct flb_http_server *session) +{ + int index; + struct flb_http_server_runtime *runtime; + + runtime = session->runtime; + if (runtime == NULL) { + return; + } + + for (index = 0; index < runtime->worker_count; index++) { + runtime->workers[index].should_exit = FLB_TRUE; + + if (runtime->workers[index].initialized == FLB_TRUE) { + pthread_join(runtime->workers[index].thread, NULL); + } + + flb_http_server_worker_context_cleanup(&runtime->workers[index]); + } + + flb_free(runtime->workers); + flb_free(runtime); + + session->runtime = NULL; +} + /* HTTP SERVER */ int flb_http_server_init(struct flb_http_server *session, int protocol_version, uint64_t flags, - flb_http_server_request_processor_callback - request_callback, + flb_http_server_request_processor_callback request_callback, char *address, unsigned short int port, struct flb_tls *tls_provider, @@ -322,36 +643,175 @@ int flb_http_server_init(struct flb_http_server *session, struct flb_config *system_context, void *user_data) { - session->status = HTTP_SERVER_UNINITIALIZED; - session->protocol_version = protocol_version; - session->flags = flags; - session->request_callback = request_callback; - session->user_data = user_data; - - session->address = address; - session->port = port; - session->tls_provider = tls_provider; - session->networking_flags = networking_flags; - session->networking_setup = networking_setup; - session->event_loop = event_loop; - session->system_context = system_context; + struct flb_http_server_options options; + + flb_http_server_options_init(&options); + + options.protocol_version = protocol_version; + options.flags = flags; + options.request_callback = request_callback; + options.user_data = user_data; + options.address = address; + options.port = port; + options.tls_provider = tls_provider; + options.networking_flags = networking_flags; + options.networking_setup = networking_setup; + options.event_loop = event_loop; + options.system_context = system_context; + + return flb_http_server_init_with_options(session, &options); +} - session->downstream = NULL; - session->buffer_max_size = HTTP_SERVER_MAXIMUM_BUFFER_SIZE; +void flb_http_server_options_init(struct flb_http_server_options *options) +{ + if (options == NULL) { + return; + } - cfl_list_init(&session->clients); + memset(options, 0, sizeof(struct flb_http_server_options)); - MK_EVENT_NEW(&session->listener_event); + options->buffer_max_size = HTTP_SERVER_MAXIMUM_BUFFER_SIZE; + options->max_connections = 0; + options->workers = 1; + options->use_caller_event_loop = FLB_TRUE; + options->reuse_port = FLB_FALSE; +} - session->status = HTTP_SERVER_INITIALIZED; +void flb_http_server_config_init(struct flb_http_server_config *config) +{ + if (config == NULL) { + return; + } + + memset(config, 0, sizeof(struct flb_http_server_config)); + + config->http2 = FLB_TRUE; + config->buffer_max_size = HTTP_SERVER_MAXIMUM_BUFFER_SIZE; + config->buffer_chunk_size = HTTP_SERVER_INITIAL_BUFFER_SIZE; + config->max_connections = 0; +} + +int flb_http_server_options_init_from_input(struct flb_http_server_options *options, + struct flb_input_instance *input_instance, + int protocol_version, + uint64_t flags, + size_t buffer_max_size, + flb_http_server_request_processor_callback + request_callback, + void *user_data) +{ + if (options == NULL || input_instance == NULL) { + return -1; + } + + flb_http_server_options_init(options); + + options->protocol_version = protocol_version; + options->flags = flags; + options->request_callback = request_callback; + options->user_data = user_data; + options->address = input_instance->host.listen; + options->port = input_instance->host.port; + options->tls_provider = input_instance->tls; + options->networking_flags = input_instance->flags; + options->networking_setup = &input_instance->net_setup; + options->event_loop = flb_input_event_loop_get(input_instance); + options->system_context = input_instance->config; + options->buffer_max_size = buffer_max_size; + options->max_connections = 0; + options->reuse_port = input_instance->net_setup.share_port; + + return 0; +} + +int flb_input_http_server_options_init(struct flb_http_server_options *options, + struct flb_input_instance *input_instance, + uint64_t flags, + flb_http_server_request_processor_callback request_callback, + void *user_data) +{ + int protocol_version; + size_t buffer_max_size; + int result; + struct flb_http_server_config *server_config; + + if (input_instance == NULL || options == NULL || + input_instance->http_server_config == NULL) { + return -1; + } + + server_config = input_instance->http_server_config; + + if (server_config != NULL && server_config->http2 == FLB_FALSE) { + protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else { + protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + } + + if (server_config != NULL && server_config->buffer_max_size > 0) { + buffer_max_size = server_config->buffer_max_size; + } + else { + buffer_max_size = HTTP_SERVER_MAXIMUM_BUFFER_SIZE; + } + + result = flb_http_server_options_init_from_input(options, + input_instance, + protocol_version, + flags, + buffer_max_size, + request_callback, + user_data); + if (result != 0) { + return result; + } + + if (server_config != NULL) { + options->max_connections = server_config->max_connections; + } return 0; } +int flb_http_server_init_with_options( + struct flb_http_server *session, + struct flb_http_server_options *options) +{ + if (session == NULL || options == NULL) { + return -1; + } + + if (options->buffer_max_size == 0) { + options->buffer_max_size = HTTP_SERVER_MAXIMUM_BUFFER_SIZE; + } + + if (options->workers <= 0) { + options->workers = 1; + } + + if (options->workers > 1) { + options->reuse_port = FLB_TRUE; + options->use_caller_event_loop = FLB_FALSE; + } + + if (options->reuse_port == FLB_TRUE && + options->networking_setup != NULL) { + options->networking_setup->share_port = FLB_TRUE; + } + + return flb_http_server_apply_options(session, options); +} + int flb_http_server_start(struct flb_http_server *session) { int result; + if (!flb_http_server_running_on_caller_context(session)) { + flb_error("[http_server] internally managed worker runtime is not enabled yet"); + return -1; + } + if (session->tls_provider != NULL) { result = flb_tls_set_alpn(session->tls_provider, "h2,http/1.0,http/1.1"); @@ -386,6 +846,19 @@ int flb_http_server_start(struct flb_http_server *session) return -1; } + if (session->cb_worker_init != NULL) { + result = session->cb_worker_init(session, + session->user_data); + + if (result != 0) { + mk_event_del(session->event_loop, &session->listener_event); + flb_downstream_destroy(session->downstream); + session->downstream = NULL; + + return result; + } + } + session->status = HTTP_SERVER_RUNNING; return 0; @@ -397,6 +870,12 @@ int flb_http_server_stop(struct flb_http_server *server) struct cfl_list *iterator; struct flb_http_server_session *session; + if (server->runtime != NULL) { + flb_http_server_runtime_stop(server); + server->status = HTTP_SERVER_STOPPED; + return 0; + } + if (server->status == HTTP_SERVER_RUNNING) { if (MK_EVENT_IS_REGISTERED((&server->listener_event))) { mk_event_del(server->event_loop, &server->listener_event); @@ -410,6 +889,10 @@ int flb_http_server_stop(struct flb_http_server *server) flb_http_server_session_destroy(session); } + if (server->cb_worker_exit != NULL) { + server->cb_worker_exit(server, server->user_data); + } + server->status = HTTP_SERVER_STOPPED; } @@ -429,6 +912,58 @@ int flb_http_server_destroy(struct flb_http_server *server) return 0; } +int flb_http_server_init_on_input(struct flb_http_server *session, + struct flb_input_instance *input_instance, + int protocol_version, + uint64_t flags, + size_t buffer_max_size, + flb_http_server_request_processor_callback request_callback, + void *user_data) +{ + int result; + struct flb_http_server_options options; + + if (session == NULL || input_instance == NULL) { + return -1; + } + + result = flb_http_server_options_init_from_input(&options, + input_instance, + protocol_version, + flags, + buffer_max_size, + request_callback, + user_data); + if (result != 0) { + return result; + } + + result = flb_http_server_init_with_options(session, &options); + + if (result != 0) { + return result; + } + + result = flb_http_server_start(session); + + if (result != 0) { + flb_http_server_destroy(session); + return result; + } + + result = 0; + + if (session->runtime == NULL && session->downstream != NULL) { + result = flb_input_downstream_set(session->downstream, input_instance); + + if (result != 0) { + flb_http_server_destroy(session); + } + } + + return result; +} + void flb_http_server_set_buffer_max_size(struct flb_http_server *server, size_t size) { diff --git a/src/http_server/flb_http_server_config_map.c b/src/http_server/flb_http_server_config_map.c new file mode 100644 index 00000000000..e86a4c8486a --- /dev/null +++ b/src/http_server/flb_http_server_config_map.c @@ -0,0 +1,98 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +struct flb_config_map flb_http_server_config_map[] = { + { + FLB_CONFIG_MAP_BOOL, "http_server.http2", "true", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, http2), + "Enable HTTP/2 support for the HTTP server" + }, + { + FLB_CONFIG_MAP_SIZE, "http_server.buffer_max_size", "4M", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, buffer_max_size), + "Set the maximum size of the HTTP request buffer" + }, + { + FLB_CONFIG_MAP_SIZE, "http_server.buffer_chunk_size", "512K", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, buffer_chunk_size), + "Set the buffer chunk size used for HTTP requests" + }, + { + FLB_CONFIG_MAP_SIZE, "http_server.max_connections", "0", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, max_connections), + "Set the maximum number of concurrent active HTTP connections. 0 means unlimited." + }, + { + FLB_CONFIG_MAP_BOOL, "http2", "true", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, http2), + "Compatibility alias for http_server.http2" + }, + { + FLB_CONFIG_MAP_SIZE, "buffer_max_size", "4M", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, buffer_max_size), + "Compatibility alias for http_server.buffer_max_size" + }, + { + FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", "512K", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, buffer_chunk_size), + "Compatibility alias for http_server.buffer_chunk_size" + }, + { + FLB_CONFIG_MAP_SIZE, "max_connections", "0", + 0, FLB_TRUE, offsetof(struct flb_http_server_config, max_connections), + "Compatibility alias for http_server.max_connections" + }, + {0} +}; + +struct mk_list *flb_http_server_get_config_map(struct flb_config *config) +{ + return flb_config_map_create(config, flb_http_server_config_map); +} + +int flb_http_server_config_map_set(struct mk_list *properties, + struct mk_list *config_map, + struct flb_http_server_config *context) +{ + return flb_config_map_set(properties, config_map, context); +} + +int flb_http_server_property_is_allowed(const char *property_name) +{ + if (property_name == NULL) { + return FLB_FALSE; + } + + if (strncasecmp("http_server.", property_name, 12) == 0) { + return FLB_TRUE; + } + + if (strcasecmp("http2", property_name) == 0 || + strcasecmp("buffer_max_size", property_name) == 0 || + strcasecmp("buffer_chunk_size", property_name) == 0 || + strcasecmp("max_connections", property_name) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} diff --git a/src/http_server/flb_http_server_http1.c b/src/http_server/flb_http_server_http1.c index f21affcf7f5..a84a9683628 100644 --- a/src/http_server/flb_http_server_http1.c +++ b/src/http_server/flb_http_server_http1.c @@ -130,6 +130,11 @@ static int http1_session_process_request(struct flb_http1_server_session *sessio return -1; } + result = flb_http_request_normalize(&session->stream.request); + if (result != 0) { + return -1; + } + switch (session->inner_request.protocol) { case MK_HTTP_PROTOCOL_09: session->stream.request.protocol_version = HTTP_PROTOCOL_VERSION_09; @@ -306,6 +311,7 @@ int flb_http1_response_commit(struct flb_http_response *response) cfl_sds_t sds_result; struct flb_http1_server_session *session; struct flb_http_stream *stream; + char *header_value; parent_session = (struct flb_http_server_session *) response->stream->parent; @@ -344,6 +350,32 @@ int flb_http1_response_commit(struct flb_http_response *response) return -4; } + header_value = flb_http_response_get_header(response, "server"); + if (header_value != NULL) { + sds_result = cfl_sds_printf(&response_buffer, + "server: %s\r\n", + header_value); + + if (sds_result == NULL) { + cfl_sds_destroy(response_buffer); + + return -5; + } + } + + header_value = flb_http_response_get_header(response, "x-http-engine"); + if (header_value != NULL) { + sds_result = cfl_sds_printf(&response_buffer, + "x-http-engine: %s\r\n", + header_value); + + if (sds_result == NULL) { + cfl_sds_destroy(response_buffer); + + return -6; + } + } + mk_list_foreach(header_iterator, &response->headers->entries) { header_entry = mk_list_entry(header_iterator, struct flb_hash_table_entry, @@ -352,7 +384,18 @@ int flb_http1_response_commit(struct flb_http_response *response) if (header_entry == NULL) { cfl_sds_destroy(response_buffer); - return -5; + return -7; + } + + if ((header_entry->key_len == strlen("server") && + strncasecmp((const char *) header_entry->key, + "server", + header_entry->key_len) == 0) || + (header_entry->key_len == strlen("x-http-engine") && + strncasecmp((const char *) header_entry->key, + "x-http-engine", + header_entry->key_len) == 0)) { + continue; } sds_result = cfl_sds_printf(&response_buffer, @@ -365,7 +408,7 @@ int flb_http1_response_commit(struct flb_http_response *response) if (sds_result == NULL) { cfl_sds_destroy(response_buffer); - return -6; + return -8; } } @@ -374,7 +417,7 @@ int flb_http1_response_commit(struct flb_http_response *response) if (sds_result == NULL) { cfl_sds_destroy(response_buffer); - return -7; + return -9; } if (response->body != NULL) { @@ -385,7 +428,7 @@ int flb_http1_response_commit(struct flb_http_response *response) if (sds_result == NULL) { cfl_sds_destroy(response_buffer); - return -8; + return -10; } response_buffer = sds_result; @@ -398,7 +441,7 @@ int flb_http1_response_commit(struct flb_http_response *response) cfl_sds_destroy(response_buffer); if (sds_result == NULL) { - return -9; + return -11; } session->parent->outgoing_data = sds_result; diff --git a/src/http_server/flb_http_server_http2.c b/src/http_server/flb_http_server_http2.c index 295c5e1be80..f6cec6ecd20 100644 --- a/src/http_server/flb_http_server_http2.c +++ b/src/http_server/flb_http_server_http2.c @@ -138,6 +138,7 @@ int flb_http2_response_commit(struct flb_http_response *response) struct flb_http2_server_session *session; struct flb_http_stream *stream; int result; + char *header_value; parent_session = (struct flb_http_server_session *) response->stream->parent; @@ -177,6 +178,24 @@ int flb_http2_response_commit(struct flb_http_response *response) header_index = 1; + header_value = flb_http_response_get_header(response, "server"); + if (header_value != NULL) { + headers[header_index].name = (uint8_t *) "server"; + headers[header_index].namelen = strlen("server"); + headers[header_index].value = (uint8_t *) header_value; + headers[header_index].valuelen = strlen(header_value); + header_index++; + } + + header_value = flb_http_response_get_header(response, "x-http-engine"); + if (header_value != NULL) { + headers[header_index].name = (uint8_t *) "x-http-engine"; + headers[header_index].namelen = strlen("x-http-engine"); + headers[header_index].value = (uint8_t *) header_value; + headers[header_index].valuelen = strlen(header_value); + header_index++; + } + mk_list_foreach(header_iterator, &response->headers->entries) { header_entry = mk_list_entry(header_iterator, struct flb_hash_table_entry, @@ -188,6 +207,17 @@ int flb_http2_response_commit(struct flb_http_response *response) return -4; } + if ((header_entry->key_len == strlen("server") && + strncasecmp((const char *) header_entry->key, + "server", + header_entry->key_len) == 0) || + (header_entry->key_len == strlen("x-http-engine") && + strncasecmp((const char *) header_entry->key, + "x-http-engine", + header_entry->key_len) == 0)) { + continue; + } + headers[header_index].name = (uint8_t *) header_entry->key; headers[header_index].namelen = header_entry->key_len; headers[header_index].value = (uint8_t *) header_entry->val; @@ -474,6 +504,11 @@ static int http2_header_callback(nghttp2_session *inner_session, if (stream->request.path == NULL) { return -1; } + + result = flb_http_request_normalize(&stream->request); + if (result != 0) { + return -1; + } } else if (flb_http_server_strncasecmp( name, name_length, ":authority", 0) == 0) { @@ -761,4 +796,3 @@ static ssize_t http2_data_source_read_callback(nghttp2_session *session, return result; } - From 328b3f13f2a824154cc7dceb602db39e2e146e27 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:23:18 -0600 Subject: [PATCH 02/42] input: add shared HTTP server integration Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input.h | 17 +++++- src/flb_input.c | 100 ++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index b072f7669fc..a6ba046f096 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -73,12 +73,14 @@ * In addition, if TLS is enabled then a * private key and certificate are required. */ +#define FLB_INPUT_HTTP_SERVER 4096 /* input uses the generic HTTP server */ /* Input status */ #define FLB_INPUT_RUNNING 1 #define FLB_INPUT_PAUSED 0 struct flb_input_instance; +struct flb_http_server_config; /* * Tests callbacks @@ -162,7 +164,6 @@ struct flb_input_plugin { char *description; struct flb_config_map *config_map; - /* Initialization */ int (*cb_init) (struct flb_input_instance *, struct flb_config *, void *); @@ -471,6 +472,10 @@ struct flb_input_instance { struct mk_list *net_config_map; struct mk_list net_properties; + struct mk_list *http_server_config_map; + struct flb_http_server_config *http_server_config; + struct mk_list http_server_properties; + struct mk_list *oauth2_jwt_config_map; struct mk_list oauth2_jwt_properties; @@ -742,6 +747,16 @@ static inline int flb_input_config_map_set(struct flb_input_instance *ins, } } + /* HTTP server properties */ + if (ins->http_server_config_map && ins->http_server_config) { + ret = flb_config_map_set(&ins->http_server_properties, + ins->http_server_config_map, + ins->http_server_config); + if (ret == -1) { + return -1; + } + } + return ret; } diff --git a/src/flb_input.c b/src/flb_input.c index 75dbd858da6..2f4a13dec42 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -263,6 +264,15 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, flb_errno(); return NULL; } + + instance->http_server_config = flb_calloc(1, sizeof(struct flb_http_server_config)); + if (!instance->http_server_config) { + flb_errno(); + flb_free(instance); + return NULL; + } + + flb_http_server_config_init(instance->http_server_config); instance->config = config; /* Get an ID */ @@ -272,6 +282,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, instance->ht_log_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 512, 0); if (!instance->ht_log_chunks) { + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -281,6 +292,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, 512, 0); if (!instance->ht_metric_chunks) { flb_hash_table_destroy(instance->ht_log_chunks); + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -291,6 +303,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, if (!instance->ht_trace_chunks) { flb_hash_table_destroy(instance->ht_log_chunks); flb_hash_table_destroy(instance->ht_metric_chunks); + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -302,6 +315,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, flb_hash_table_destroy(instance->ht_log_chunks); flb_hash_table_destroy(instance->ht_metric_chunks); flb_hash_table_destroy(instance->ht_trace_chunks); + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -319,6 +333,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, ctx = flb_calloc(1, sizeof(struct flb_plugin_input_proxy_context)); if (!ctx) { flb_errno(); + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -366,12 +381,14 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, /* Initialize properties list */ flb_kv_init(&instance->properties); flb_kv_init(&instance->net_properties); + flb_kv_init(&instance->http_server_properties); flb_kv_init(&instance->oauth2_jwt_properties); /* Plugin use networking */ if (plugin->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) { ret = flb_net_host_set(plugin->name, &instance->host, input); if (ret != 0) { + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -648,6 +665,18 @@ int flb_input_set_property(struct flb_input_instance *ins, } kv->val = tmp; } + else if ((ins->p->flags & FLB_INPUT_HTTP_SERVER) != 0 && + flb_http_server_property_is_allowed(k) == FLB_TRUE && + tmp != NULL) { + kv = flb_kv_item_create(&ins->http_server_properties, (char *) k, NULL); + if (!kv) { + if (tmp) { + flb_sds_destroy(tmp); + } + return -1; + } + kv->val = tmp; + } #ifdef FLB_HAVE_TLS else if (prop_key_check("tls", k, len) == 0 && tmp) { @@ -890,6 +919,7 @@ void flb_input_instance_destroy(struct flb_input_instance *ins) /* release properties */ flb_kv_release(&ins->properties); flb_kv_release(&ins->net_properties); + flb_kv_release(&ins->http_server_properties); flb_kv_release(&ins->oauth2_jwt_properties); @@ -921,6 +951,14 @@ void flb_input_instance_destroy(struct flb_input_instance *ins) flb_config_map_destroy(ins->net_config_map); } + if (ins->http_server_config_map) { + flb_config_map_destroy(ins->http_server_config_map); + } + + if (ins->http_server_config) { + flb_free(ins->http_server_config); + } + if (ins->oauth2_jwt_config_map) { flb_config_map_destroy(ins->oauth2_jwt_config_map); } @@ -1073,9 +1111,33 @@ int flb_input_plugin_property_check(struct flb_input_instance *ins, struct flb_config *config) { int ret = 0; + struct mk_list *tmp; + struct mk_list *head; struct mk_list *config_map; + struct flb_kv *kv; + struct flb_kv *moved; struct flb_input_plugin *p = ins->p; + if ((p->flags & FLB_INPUT_HTTP_SERVER) != 0) { + mk_list_foreach_safe(head, tmp, &ins->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + + if (flb_http_server_property_is_allowed(kv->key) == FLB_FALSE) { + continue; + } + + moved = flb_kv_item_create(&ins->http_server_properties, kv->key, NULL); + if (moved == NULL) { + return -1; + } + + moved->val = kv->val; + kv->val = NULL; + mk_list_del(&kv->_head); + flb_kv_item_destroy(kv); + } + } + if (p->config_map) { /* * Create a dynamic version of the configmap that will be used by the specific @@ -1106,7 +1168,7 @@ int flb_input_plugin_property_check(struct flb_input_instance *ins, } int flb_input_oauth2_jwt_property_check(struct flb_input_instance *ins, - struct flb_config *config) + struct flb_config *config) { int ret = 0; @@ -1137,6 +1199,35 @@ int flb_input_oauth2_jwt_property_check(struct flb_input_instance *ins, return 0; } +static int flb_input_http_server_property_check(struct flb_input_instance *ins, + struct flb_config *config) +{ + int ret = 0; + + if (ins->http_server_config_map == NULL) { + ins->http_server_config_map = flb_http_server_get_config_map(config); + if (!ins->http_server_config_map) { + flb_input_instance_destroy(ins); + return -1; + } + } + + if (mk_list_size(&ins->http_server_properties) > 0) { + ret = flb_config_map_properties_check(ins->p->name, + &ins->http_server_properties, + ins->http_server_config_map); + if (ret == -1) { + if (config->program_name) { + flb_helper("try the command: %s -i %s -h\n", + config->program_name, ins->p->name); + } + return -1; + } + } + + return 0; +} + int flb_input_instance_init(struct flb_input_instance *ins, struct flb_config *config) { @@ -1340,6 +1431,13 @@ int flb_input_instance_init(struct flb_input_instance *ins, } } + if ((p->flags & FLB_INPUT_HTTP_SERVER) != 0 || + mk_list_size(&ins->http_server_properties) > 0) { + if (flb_input_http_server_property_check(ins, config) == -1) { + return -1; + } + } + #ifdef FLB_HAVE_TLS if (ins->use_tls == FLB_TRUE) { if ((p->flags & FLB_INPUT_NET_SERVER) != 0) { From f7016b71846da4dcda12d1c6e90ef942fbb154c8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:23:30 -0600 Subject: [PATCH 03/42] output: add shared HTTP server integration Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_output.h | 16 ++++++ src/flb_output.c | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 267b6ff0dbd..6091f497464 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -83,6 +83,7 @@ int flb_chunk_trace_output(struct flb_chunk_trace *trace, struct flb_output_inst #define FLB_OUTPUT_NO_MULTIPLEX 512 /* run one task at a time, one task per flush */ #define FLB_OUTPUT_PRIVATE 1024 #define FLB_OUTPUT_SYNCHRONOUS 2048 /* run one task at a time, no flush cycle limit */ +#define FLB_OUTPUT_HTTP_SERVER 4096 /* output uses the generic HTTP server */ /* @@ -106,6 +107,7 @@ int flb_chunk_trace_output(struct flb_chunk_trace *trace, struct flb_output_inst const char *tag = event_chunk->tag; struct flb_output_flush; +struct flb_http_server_config; /* * Tests callbacks @@ -434,6 +436,10 @@ struct flb_output_instance { struct mk_list *net_config_map; struct mk_list net_properties; + struct mk_list *http_server_config_map; + struct flb_http_server_config *http_server_config; + struct mk_list http_server_properties; + struct mk_list *oauth2_config_map; struct mk_list oauth2_properties; @@ -1325,6 +1331,16 @@ static inline int flb_output_config_map_set(struct flb_output_instance *ins, } } + /* HTTP server properties */ + if (ins->http_server_config_map && ins->http_server_config) { + ret = flb_config_map_set(&ins->http_server_properties, + ins->http_server_config_map, + ins->http_server_config); + if (ret == -1) { + return -1; + } + } + /* OAuth2 properties are validated but not automatically applied here. * Plugins should call flb_config_map_set() with &ctx->oauth2_config * in their init callback after calling flb_output_config_map_set(). */ diff --git a/src/flb_output.c b/src/flb_output.c index 6469700071a..837f0111379 100644 --- a/src/flb_output.c +++ b/src/flb_output.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -166,6 +167,7 @@ static void flb_output_free_properties(struct flb_output_instance *ins) flb_kv_release(&ins->properties); flb_kv_release(&ins->net_properties); + flb_kv_release(&ins->http_server_properties); flb_kv_release(&ins->oauth2_properties); #ifdef FLB_HAVE_TLS @@ -512,6 +514,14 @@ int flb_output_instance_destroy(struct flb_output_instance *ins) flb_config_map_destroy(ins->net_config_map); } + if (ins->http_server_config_map) { + flb_config_map_destroy(ins->http_server_config_map); + } + + if (ins->http_server_config) { + flb_free(ins->http_server_config); + } + if (ins->oauth2_config_map) { flb_config_map_destroy(ins->oauth2_config_map); } @@ -694,6 +704,15 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, return NULL; } + instance->http_server_config = flb_calloc(1, sizeof(struct flb_http_server_config)); + if (!instance->http_server_config) { + flb_errno(); + flb_free(instance); + return NULL; + } + + flb_http_server_config_init(instance->http_server_config); + /* Initialize event type, if not set, default to FLB_OUTPUT_LOGS */ if (plugin->event_type == 0) { instance->event_type = FLB_OUTPUT_LOGS; @@ -720,6 +739,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { flb_task_queue_destroy(instance->singleplex_queue); } + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -736,6 +756,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { flb_task_queue_destroy(instance->singleplex_queue); } + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -798,6 +819,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { flb_task_queue_destroy(instance->singleplex_queue); } + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -808,6 +830,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, if (instance->flags & FLB_OUTPUT_SYNCHRONOUS) { instance->singleplex_queue = flb_task_queue_create(); if (!instance->singleplex_queue) { + flb_free(instance->http_server_config); flb_free(instance); flb_errno(); return NULL; @@ -816,6 +839,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, flb_kv_init(&instance->properties); flb_kv_init(&instance->net_properties); + flb_kv_init(&instance->http_server_properties); flb_kv_init(&instance->oauth2_properties); mk_list_init(&instance->upstreams); mk_list_init(&instance->flush_list); @@ -956,6 +980,18 @@ int flb_output_set_property(struct flb_output_instance *ins, } kv->val = tmp; } + else if ((ins->p->flags & FLB_OUTPUT_HTTP_SERVER) != 0 && + flb_http_server_property_is_allowed(k) == FLB_TRUE && + tmp != NULL) { + kv = flb_kv_item_create(&ins->http_server_properties, (char *) k, NULL); + if (!kv) { + if (tmp) { + flb_sds_destroy(tmp); + } + return -1; + } + kv->val = tmp; + } #ifdef FLB_HAVE_HTTP_CLIENT_DEBUG else if (strncasecmp("_debug.http.", k, 12) == 0 && tmp) { ret = flb_http_client_debug_property_is_valid((char *) k, tmp); @@ -1197,13 +1233,65 @@ int flb_output_oauth2_property_check(struct flb_output_instance *ins, return 0; } +static int flb_output_http_server_property_check(struct flb_output_instance *ins, + struct flb_config *config) +{ + int ret = 0; + + if (ins->http_server_config_map == NULL) { + ins->http_server_config_map = flb_http_server_get_config_map(config); + if (!ins->http_server_config_map) { + return -1; + } + } + + if (mk_list_size(&ins->http_server_properties) > 0) { + ret = flb_config_map_properties_check(ins->p->name, + &ins->http_server_properties, + ins->http_server_config_map); + if (ret == -1) { + if (config->program_name) { + flb_helper("try the command: %s -o %s -h\n", + config->program_name, ins->p->name); + } + return -1; + } + } + + return 0; +} + int flb_output_plugin_property_check(struct flb_output_instance *ins, struct flb_config *config) { int ret = 0; + struct mk_list *tmp; + struct mk_list *head; struct mk_list *config_map; + struct flb_kv *kv; + struct flb_kv *moved; struct flb_output_plugin *p = ins->p; + if ((p->flags & FLB_OUTPUT_HTTP_SERVER) != 0) { + mk_list_foreach_safe(head, tmp, &ins->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + + if (flb_http_server_property_is_allowed(kv->key) == FLB_FALSE) { + continue; + } + + moved = flb_kv_item_create(&ins->http_server_properties, kv->key, NULL); + if (moved == NULL) { + return -1; + } + + moved->val = kv->val; + kv->val = NULL; + mk_list_del(&kv->_head); + flb_kv_item_destroy(kv); + } + } + if (p->config_map) { /* * Create a dynamic version of the configmap that will be used by the specific @@ -1560,6 +1648,14 @@ int flb_output_init_all(struct flb_config *config) } } + if ((p->flags & FLB_OUTPUT_HTTP_SERVER) != 0 || + mk_list_size(&ins->http_server_properties) > 0) { + if (flb_output_http_server_property_check(ins, config) == -1) { + flb_output_instance_destroy(ins); + return -1; + } + } + /* Initialize plugin through it 'init callback' */ ret = p->cb_init(ins, config, ins->data); if (ret == -1) { From 9de25f9e351b6e93e33a7826e5417cedf3798e57 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:23:45 -0600 Subject: [PATCH 04/42] in_http: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/in_http/CMakeLists.txt | 1 - plugins/in_http/http.c | 163 ++----- plugins/in_http/http.h | 16 - plugins/in_http/http_config.c | 92 +--- plugins/in_http/http_prot.c | 813 ++------------------------------- plugins/in_http/http_prot.h | 8 - 6 files changed, 81 insertions(+), 1012 deletions(-) diff --git a/plugins/in_http/CMakeLists.txt b/plugins/in_http/CMakeLists.txt index 69ebeab7148..6e7826c447d 100644 --- a/plugins/in_http/CMakeLists.txt +++ b/plugins/in_http/CMakeLists.txt @@ -4,7 +4,6 @@ endif() set(src http.c - http_conn.c http_prot.c http_config.c ) diff --git a/plugins/in_http/http.c b/plugins/in_http/http.c index 333a7817ad4..bf42f72eaba 100644 --- a/plugins/in_http/http.c +++ b/plugins/in_http/http.c @@ -23,52 +23,15 @@ #include #include "http.h" -#include "http_conn.h" #include "http_prot.h" #include "http_config.h" -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_http_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct http_conn *conn; - struct flb_http *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = http_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - static int in_http_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_http *ctx; + struct flb_http_server_options http_server_options; (void) data; @@ -78,8 +41,6 @@ static int in_http_init(struct flb_input_instance *ins, return -1; } - ctx->collector_id = -1; - /* Populate context with config map defaults and incoming properties */ ret = flb_input_config_map_set(ins, (void *) ctx); if (ret == -1) { @@ -110,72 +71,42 @@ static int in_http_init(struct flb_input_instance *ins, /* Set the context */ flb_input_set_context(ins, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_VERSION_AUTODETECT, - (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), - NULL, - ins->host.listen, - ins->host.port, - ins->tls, - ins->flags, - &ins->net_setup, - flb_input_event_loop_get(ins), - ins->config, - (void *) ctx); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not initialize http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - http_config_destroy(ctx); - - return -1; + ret = flb_input_http_server_options_init(&http_server_options, + ins, + (FLB_HTTP_SERVER_FLAG_KEEPALIVE | + FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), + http_prot_handle_ng, + ctx); + if (ret == 0) { + ret = flb_http_server_init_with_options(&ctx->http_server, + &http_server_options); + + if (ret == 0) { + ret = flb_http_server_start(&ctx->http_server); } - ret = flb_http_server_start(&ctx->http_server); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not start http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - http_config_destroy(ctx); - - return -1; + if (ret == 0) { + ret = flb_input_downstream_set(ctx->http_server.downstream, ins); } - - flb_http_server_set_buffer_max_size(&ctx->http_server, ctx->buffer_max_size); - - ctx->http_server.request_callback = http_prot_handle_ng; - - flb_input_downstream_set(ctx->http_server.downstream, ctx->ins); } - else { - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - http_config_destroy(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start http server on %s:%u. Aborting", + ins->host.listen, ins->host.port); - return -1; - } + http_config_destroy(ctx); - flb_input_downstream_set(ctx->downstream, ctx->ins); + return -1; } + flb_plg_info(ctx->ins, + "listening on %s:%u with %i worker%s", + ins->host.listen, + ins->host.port, + ctx->http_server.workers, + ctx->http_server.workers == 1 ? "" : "s"); + if (ctx->successful_response_code != 200 && ctx->successful_response_code != 201 && ctx->successful_response_code != 204) { @@ -184,22 +115,6 @@ static int in_http_init(struct flb_input_instance *ins, ctx->successful_response_code = 201; } - if (!ctx->enable_http2) { - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_http_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - http_config_destroy(ctx); - - return -1; - } - - ctx->collector_id = ret; - } - return 0; } @@ -220,12 +135,6 @@ static int in_http_exit(void *data, struct flb_config *config) /* Configuration properties map */ static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "http2", "true", - 0, FLB_TRUE, offsetof(struct flb_http, enable_http2), - "Enable HTTP/2 support." - }, - { FLB_CONFIG_MAP_BOOL, "add_remote_addr", "false", 0, FLB_TRUE, offsetof(struct flb_http, add_remote_addr), @@ -238,18 +147,6 @@ static struct flb_config_map config_map[] = { "Key name for the remote address field added to the record." }, - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_http, buffer_max_size), - "Set the maximum buffer size to store incoming data." - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_http, buffer_chunk_size), - "Set the initial buffer size to store incoming data." - }, - { FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_http, success_headers), @@ -278,11 +175,11 @@ struct flb_input_plugin in_http_plugin = { .description = "HTTP", .cb_init = in_http_init, .cb_pre_run = NULL, - .cb_collect = in_http_collect, + .cb_collect = NULL, .cb_flush_buf = NULL, .cb_pause = NULL, .cb_resume = NULL, .cb_exit = in_http_exit, .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS + .flags = FLB_INPUT_NET_SERVER | FLB_INPUT_HTTP_SERVER | FLB_IO_OPT_TLS }; diff --git a/plugins/in_http/http.h b/plugins/in_http/http.h index 0aab823d977..92ec48229bd 100644 --- a/plugins/in_http/http.h +++ b/plugins/in_http/http.h @@ -20,7 +20,6 @@ #ifndef FLB_IN_HTTP_H #define FLB_IN_HTTP_H -#include #include #include #include @@ -28,7 +27,6 @@ #include #include -#include #include #define HTTP_BUFFER_MAX_SIZE "4M" @@ -52,25 +50,11 @@ struct flb_http { int add_remote_addr; const char *remote_addr_key; - /* New gen HTTP server */ - int enable_http2; struct flb_http_server http_server; struct flb_oauth2_jwt_cfg oauth2_cfg; struct flb_oauth2_jwt_ctx *oauth2_ctx; - /* Legacy HTTP server */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - flb_sds_t success_headers_str; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - struct mk_server *server; - - int collector_id; }; diff --git a/plugins/in_http/http_config.c b/plugins/in_http/http_config.c index 9d7acfa9d9f..53252611d25 100644 --- a/plugins/in_http/http_config.c +++ b/plugins/in_http/http_config.c @@ -22,15 +22,9 @@ #include "http.h" #include "http_config.h" -#include "http_conn.h" -#include "http_config.h" struct flb_http *http_config_create(struct flb_input_instance *ins) { - struct mk_list *header_iterator; - struct flb_slist_entry *header_value; - struct flb_slist_entry *header_name; - struct flb_config_map_val *header_pair; char port[8]; int ret; struct flb_http *ctx; @@ -41,7 +35,6 @@ struct flb_http *http_config_create(struct flb_input_instance *ins) return NULL; } ctx->ins = ins; - mk_list_init(&ctx->connections); ctx->oauth2_cfg.jwks_refresh_interval = 300; @@ -69,20 +62,6 @@ struct flb_http *http_config_create(struct flb_input_instance *ins) snprintf(port, sizeof(port) - 1, "%d", ins->host.port); ctx->tcp_port = flb_strdup(port); - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - if (!ctx->server) { - flb_errno(); - http_config_destroy(ctx); - return NULL; - } - - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - ret = flb_log_event_encoder_init(&ctx->log_encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); @@ -92,13 +71,6 @@ struct flb_http *http_config_create(struct flb_input_instance *ins) return NULL; } - ctx->success_headers_str = flb_sds_create_size(1); - - if (ctx->success_headers_str == NULL) { - http_config_destroy(ctx); - return NULL; - } - /* Create record accessor for tag_key if specified */ if (ctx->tag_key) { ctx->ra_tag_key = flb_ra_create(ctx->tag_key, FLB_TRUE); @@ -109,44 +81,6 @@ struct flb_http *http_config_create(struct flb_input_instance *ins) } } - flb_config_map_foreach(header_iterator, header_pair, ctx->success_headers) { - header_name = mk_list_entry_first(header_pair->val.list, - struct flb_slist_entry, - _head); - - header_value = mk_list_entry_last(header_pair->val.list, - struct flb_slist_entry, - _head); - - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_name->str, - flb_sds_len(header_name->str)); - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - ": ", - 2); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - header_value->str, - flb_sds_len(header_value->str)); - } - - if (ret == 0) { - ret = flb_sds_cat_safe(&ctx->success_headers_str, - "\r\n", - 2); - } - - if (ret != 0) { - http_config_destroy(ctx); - - return NULL; - } - } - return ctx; } @@ -156,32 +90,8 @@ int http_config_destroy(struct flb_http *ctx) flb_ra_destroy(ctx->ra_tag_key); } - /* release all connections */ - http_conn_release_all(ctx); - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->server) { - flb_free(ctx->server); - } - - if (ctx->enable_http2) { - flb_http_server_destroy(&ctx->http_server); - } - - if (ctx->success_headers_str != NULL) { - flb_sds_destroy(ctx->success_headers_str); - } + flb_http_server_destroy(&ctx->http_server); if (ctx->oauth2_ctx) { flb_oauth2_jwt_context_destroy(ctx->oauth2_ctx); diff --git a/plugins/in_http/http_prot.c b/plugins/in_http/http_prot.c index 71176f71952..b9fd0efe037 100644 --- a/plugins/in_http/http_prot.c +++ b/plugins/in_http/http_prot.c @@ -18,15 +18,11 @@ */ #include -#include #include #include #include -#include -#include -#include #include #include @@ -34,16 +30,49 @@ #include #include "http.h" -#include "http_conn.h" #define HTTP_CONTENT_JSON 0 #define HTTP_CONTENT_URLENCODED 1 -static int http_header_lookup(int version, void *ptr, char *key, +static int http_header_lookup(struct flb_http_request *request, char *key, char **val, size_t *val_len); static int process_pack_ng(struct flb_http *ctx, flb_sds_t tag, char *buf, size_t size, void *request); +static int content_type_is_json(const char *content_type) +{ + size_t length; + + if (content_type == NULL) { + return FLB_FALSE; + } + + length = strlen(content_type); + + if (length == 16 && + strncasecmp(content_type, "application/json", 16) == 0) { + return FLB_TRUE; + } + + if (length > 16 && + (strncasecmp(content_type, "application/json;", 17) == 0 || + strncasecmp(content_type, "application/json ", 17) == 0)) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int content_type_is_urlencoded(const char *content_type) +{ + if (content_type == NULL) { + return FLB_FALSE; + } + + return strcasecmp(content_type, + "application/x-www-form-urlencoded") == 0; +} + static inline char hex2nibble(char c) { if ((c >= 0x30) && (c <= '9')) { @@ -94,90 +123,6 @@ static int sds_uri_decode(flb_sds_t s) return 0; } -static int send_response(struct http_conn *conn, int http_status, char *message) -{ - struct flb_http *context; - size_t sent; - int len; - flb_sds_t out; - - context = (struct flb_http *) conn->ctx; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 413) { - flb_sds_printf(&out, - "HTTP/1.1 413 Request Entity Too Large\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message ? message : ""); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - else if (http_status == 401) { - flb_sds_printf(&out, - "HTTP/1.1 401 Unauthorized\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message ? message : ""); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - static void sanitize_tag(flb_sds_t tag) { size_t i; @@ -229,15 +174,14 @@ static flb_sds_t tag_key(struct flb_http *ctx, msgpack_object *map) } /* Extract the client IP address from the X-Forwarded-For header */ -static flb_sds_t get_remote_addr(void *request, int version) +static flb_sds_t get_remote_addr(struct flb_http_request *request) { int ret; char *ptr = NULL; size_t len = 0; flb_sds_t remote_addr; - ret = http_header_lookup(version, - request, + ret = http_header_lookup(request, "X-Forwarded-For", &ptr, &len); @@ -367,200 +311,6 @@ static int process_pack_record(struct flb_http *ctx, struct flb_time *tm, return 0; } -int process_pack(struct flb_http *ctx, flb_sds_t tag, char *buf, size_t size, void *request) -{ - int ret; - size_t off = 0; - msgpack_unpacked result; - struct flb_time tm; - int i = 0; - msgpack_object *obj; - msgpack_object record; - flb_sds_t tag_from_record = NULL; - - flb_sds_t remote_addr = NULL; - msgpack_unpacked appended_result; - msgpack_sbuffer appended_sbuf; - int appended_initialized = 0; - - flb_time_get(&tm); - - if (ctx->add_remote_addr == FLB_TRUE && ctx->remote_addr_key != NULL) { - remote_addr = get_remote_addr(request, HTTP_PROTOCOL_VERSION_11); - } - - msgpack_unpacked_init(&result); - while (msgpack_unpack_next(&result, buf, size, &off) == MSGPACK_UNPACK_SUCCESS) { - if (result.data.type == MSGPACK_OBJECT_MAP) { - obj = &result.data; - if (remote_addr != NULL && flb_sds_len(remote_addr) > 0) { - if (!appended_initialized) { - /* doing this only once, since it can be cleared and reused */ - msgpack_sbuffer_init(&appended_sbuf); - appended_initialized = 1; - } - else if (appended_result.zone != NULL) { - msgpack_unpacked_destroy(&appended_result); - } - - /* if we fail to append, we just continue with the original object */ - msgpack_unpacked_init(&appended_result); - if (append_remote_addr(obj, &appended_result, &appended_sbuf, ctx, remote_addr) == 0) { - obj = &appended_result.data; - } - } - - tag_from_record = NULL; - if (ctx->tag_key) { - tag_from_record = tag_key(ctx, obj); - } - - if (tag_from_record) { - ret = process_pack_record(ctx, &tm, tag_from_record, obj); - flb_sds_destroy(tag_from_record); - } - else if (tag) { - ret = process_pack_record(ctx, &tm, tag, obj); - } - else { - ret = process_pack_record(ctx, &tm, NULL, obj); - } - - if (ret != 0) { - goto log_event_error; - } - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - else if (result.data.type == MSGPACK_OBJECT_ARRAY) { - obj = &result.data; - for (i = 0; i < obj->via.array.size; i++) { - record = obj->via.array.ptr[i]; - if (record.type == MSGPACK_OBJECT_MAP && - remote_addr != NULL && flb_sds_len(remote_addr) > 0) { - if (!appended_initialized) { - /* doing this only once, since it can be cleared and reused */ - msgpack_sbuffer_init(&appended_sbuf); - appended_initialized = 1; - } - else if (appended_result.zone != NULL) { - msgpack_unpacked_destroy(&appended_result); - } - - /* if we fail to append, we just continue with the original object */ - msgpack_unpacked_init(&appended_result); - if (append_remote_addr(&record, &appended_result, &appended_sbuf, ctx, remote_addr) == 0) { - record = appended_result.data; - } - } - - tag_from_record = NULL; - if (ctx->tag_key) { - tag_from_record = tag_key(ctx, &record); - } - - if (tag_from_record) { - ret = process_pack_record(ctx, &tm, tag_from_record, &record); - flb_sds_destroy(tag_from_record); - } - else if (tag) { - ret = process_pack_record(ctx, &tm, tag, &record); - } - else { - ret = process_pack_record(ctx, &tm, NULL, &record); - } - - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - goto log_event_error; - } - - /* TODO : Optimize this - * - * This is wasteful, considering that we are emitting a series - * of records we should start and commit each one and then - * emit them all at once after the loop. - */ - - flb_log_event_encoder_reset(&ctx->log_encoder); - } - - break; - } - else { - flb_plg_error(ctx->ins, "skip record from invalid type: %i", - result.data.type); - - msgpack_unpacked_destroy(&result); - if (remote_addr != NULL) { - flb_sds_destroy(remote_addr); - } - - return -1; - } - } - - msgpack_unpacked_destroy(&result); - if (appended_initialized) { - msgpack_unpacked_destroy(&appended_result); - msgpack_sbuffer_destroy(&appended_sbuf); - } - - if (remote_addr != NULL) { - flb_sds_destroy(remote_addr); - } - - return 0; - -log_event_error: - msgpack_unpacked_destroy(&result); - if (appended_initialized) { - msgpack_unpacked_destroy(&appended_result); - msgpack_sbuffer_destroy(&appended_sbuf); - } - if (remote_addr != NULL) { - flb_sds_destroy(remote_addr); - } - flb_plg_error(ctx->ins, "Error encoding record : %d", ret); - return ret; -} - -static ssize_t parse_payload_json(struct flb_http *ctx, flb_sds_t tag, - char *payload, size_t size, void *request) -{ - int ret; - int out_size; - char *pack; - struct flb_pack_state pack_state; - - /* Initialize packer */ - flb_pack_state_init(&pack_state); - - /* Pack JSON as msgpack */ - ret = flb_pack_json_state(payload, size, - &pack, &out_size, &pack_state); - flb_pack_state_reset(&pack_state); - - /* Handle exceptions */ - if (ret == FLB_ERR_JSON_PART) { - flb_plg_warn(ctx->ins, "JSON data is incomplete, skipping"); - return -1; - } - else if (ret == FLB_ERR_JSON_INVAL) { - flb_plg_warn(ctx->ins, "invalid JSON message, skipping"); - return -1; - } - else if (ret == -1) { - flb_plg_warn(ctx->ins, "error parsing JSON message, skipping"); - return -1; - } - - /* Process the packaged JSON and return the last byte used */ - ret = process_pack(ctx, tag, pack, out_size, request); - flb_free(pack); - - return ret; -} - static ssize_t parse_payload_urlencoded(struct flb_http *ctx, flb_sds_t tag, char *payload, size_t size, void *request) { @@ -657,12 +407,7 @@ static ssize_t parse_payload_urlencoded(struct flb_http *ctx, flb_sds_t tag, msgpack_pack_str_body(&pck, vals[i], flb_sds_len(vals[i])); } - if (ctx->enable_http2) { - ret = process_pack_ng(ctx, tag, sbuf.data, sbuf.size, request); - } - else { - ret = process_pack(ctx, tag, sbuf.data, sbuf.size, request); - } + ret = process_pack_ng(ctx, tag, sbuf.data, sbuf.size, request); decode_error: for (idx = 0; idx < mk_list_size(kvs); idx++) { @@ -683,484 +428,25 @@ static ssize_t parse_payload_urlencoded(struct flb_http *ctx, flb_sds_t tag, return ret; } -/* - * We use two backends for HTTP parsing and it depends on the version of the - * protocol: - * - * http/1.x: we use Monkey HTTP parser: struct mk_http_session.parser - */ -static int http_header_lookup(int version, void *ptr, char *key, +static int http_header_lookup(struct flb_http_request *request, char *key, char **val, size_t *val_len) { - int key_len; - - /* HTTP/1.1 */ - struct mk_list *head; - struct mk_http_session *session; - struct mk_http_request *request_11; - struct mk_http_header *header; - - /* HTTP/2.0 */ char *value; - struct flb_http_request *request_20; - - if (!key) { - return -1; - } - - key_len = strlen(key); - if (key_len <= 0) { - return -1; - } - - if (version <= HTTP_PROTOCOL_VERSION_11) { - if (!ptr) { - return -1; - } - - request_11 = (struct mk_http_request *) ptr; - session = request_11->session; - mk_list_foreach(head, &session->parser.header_list) { - header = mk_list_entry(head, struct mk_http_header, _head); - if (header->key.len == key_len && - strncasecmp(header->key.data, key, key_len) == 0) { - *val = header->val.data; - *val_len = header->val.len; - return 0; - } - } - return -1; - } - else if (version == HTTP_PROTOCOL_VERSION_20) { - request_20 = ptr; - if (!request_20) { - return -1; - } - - value = flb_http_request_get_header(request_20, key); - if (!value) { - return -1; - } - - *val = value; - *val_len = strlen(value); - return 0; - } - - return -1; -} - -static \ -int uncompress_zlib(struct flb_http *ctx, - char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - flb_plg_warn(ctx->ins, "zlib decompression is not supported"); - return 0; -} - -static \ -int uncompress_zstd(struct flb_http *ctx, - char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_zstd_uncompress(input_buffer, - input_size, - (void *) output_buffer, - output_size); - - if (ret != 0) { - flb_plg_error(ctx->ins, "zstd decompression failed"); - return -1; - } - - return 1; -} - -static \ -int uncompress_deflate(struct flb_http *ctx, - char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - flb_plg_warn(ctx->ins, "deflate decompression is not supported"); - return 0; -} - -static \ -int uncompress_snappy(struct flb_http *ctx, - char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_snappy_uncompress_framed_data(input_buffer, - input_size, - output_buffer, - output_size); - - if (ret != 0) { - flb_plg_error(ctx->ins, "snappy decompression failed"); - return -1; - } - - return 1; -} - -static \ -int uncompress_gzip(struct flb_http *ctx, - char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_gzip_uncompress(input_buffer, - input_size, - (void *) output_buffer, - output_size); - - if (ret == -1) { - flb_error("[opentelemetry] gzip decompression failed"); - - return -1; - } - - return 1; -} - -/* Used for HTTP/1.1 */ -static int http_prot_uncompress(struct flb_http *ctx, - struct mk_http_request *request, - char **output_buffer, - size_t *output_size) -{ - int ret = 0; - char *body; - size_t body_size; - char *encoding; - size_t encoding_len; - - *output_buffer = NULL; - *output_size = 0; - - /* get the Content-Encoding */ - ret = http_header_lookup(HTTP_PROTOCOL_VERSION_11, - request, - "Content-Encoding", - &encoding, &encoding_len); - - /* FYI: no encoding was found, assume no payload compression */ - if (ret < 0) { - return 0; - } - - /* set the payload pointers */ - body = request->data.data; - body_size = request->data.len; - - if (strncasecmp(encoding, "gzip", 4) == 0 && encoding_len == 4) { - return uncompress_gzip(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "zlib", 4) == 0 && encoding_len == 4) { - return uncompress_zlib(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "zstd", 4) == 0 && encoding_len == 4) { - return uncompress_zstd(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "snappy", 6) == 0 && encoding_len == 6) { - return uncompress_snappy(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "deflate", 7) == 0 && encoding_len == 7) { - return uncompress_deflate(ctx, - output_buffer, output_size, - body, body_size); - } - else { - return -2; - } - - return 0; -} - -static int process_payload(struct flb_http *ctx, struct http_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int ret = -1; - int type = -1; - char *original_data; - size_t original_data_size; - char *out_chunked = NULL; - size_t out_chunked_size; - struct mk_http_header *header; - char *uncompressed_data = NULL; - size_t uncompressed_data_size = 0; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - - if (((header->val.len == 16 && strncasecmp(header->val.data, "application/json", 16) == 0)) || - ((header->val.len > 16 && (strncasecmp(header->val.data, "application/json ", 17) == 0)) || - strncasecmp(header->val.data, "application/json;", 17) == 0)) { - type = HTTP_CONTENT_JSON; - } - - if (header->val.len == 33 && - strncasecmp(header->val.data, "application/x-www-form-urlencoded", 33) == 0) { - type = HTTP_CONTENT_URLENCODED; - } - if (type == -1) { - send_response(conn, 400, "error: invalid 'Content-Type'\n"); + if (request == NULL || key == NULL) { return -1; } - if (request->data.len <= 0 && !mk_http_parser_is_content_chunked(&session->parser)) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - /* content: check if the data comes in chunks (transfer-encoding: chunked) */ - if (mk_http_parser_is_content_chunked(&session->parser)) { - ret = mk_http_parser_chunked_decode(&session->parser, - conn->buf_data, - conn->buf_len, - &out_chunked, - &out_chunked_size); - - if (ret == -1) { - send_response(conn, 400, "error: invalid chunked data\n"); - return -1; - } - - /* link the decoded data */ - original_data = request->data.data; - original_data_size = request->data.len; - - request->data.data = out_chunked; - request->data.len = out_chunked_size; - } - - /* - * HTTP/1.x can have the payload compressed, we try to detect based on the - * Content-Encoding header. - */ - ret = http_prot_uncompress(ctx, - request, - &uncompressed_data, - &uncompressed_data_size); - - if (ret > 0) { - request->data.data = uncompressed_data; - request->data.len = uncompressed_data_size; - } - - if (type == HTTP_CONTENT_JSON) { - ret = parse_payload_json(ctx, tag, request->data.data, request->data.len, request); - } - else if (type == HTTP_CONTENT_URLENCODED) { - ret = parse_payload_urlencoded(ctx, tag, request->data.data, request->data.len, request); - } - - if (uncompressed_data != NULL) { - flb_free(uncompressed_data); - } - - if (out_chunked) { - mk_mem_free(out_chunked); - request->data.data = original_data; - request->data.len = original_data_size; - } - - if (ret != 0) { - send_response(conn, 400, "error: invalid payload\n"); + value = flb_http_request_get_header(request, key); + if (!value) { return -1; } + *val = value; + *val_len = strlen(value); return 0; } -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int ret; - int len; - int auth_status; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - tag = flb_sds_create_size(len); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* New tag skipping the URI '/' */ - flb_sds_cat_safe(&tag, uri + 1, len - 1); - - sanitize_tag(tag); - } - - mk_mem_free(uri); - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - if (ctx->oauth2_ctx) { - header = &session->parser.headers[MK_HEADER_AUTHORIZATION]; - if (header->type == MK_HEADER_AUTHORIZATION) { - auth_status = flb_oauth2_jwt_validate(ctx->oauth2_ctx, - header->val.data, - header->val.len); - } - else { - auth_status = FLB_OAUTH2_JWT_ERR_MISSING_AUTH_HEADER; - } - - if (auth_status != FLB_OAUTH2_JWT_OK) { - flb_plg_error(ctx->ins, "OAuth2 validation failed: %s (rejecting request with 401)", - flb_oauth2_jwt_status_message(auth_status)); - flb_sds_destroy(tag); - send_response(conn, 401, NULL); - return -1; - } - } - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method != MK_METHOD_POST) { - flb_sds_destroy(tag); - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - ret = process_payload(ctx, conn, tag, session, request); - flb_sds_destroy(tag); - - if (ret == 0) { - send_response(conn, ctx->successful_response_code, NULL); - } - else { - send_response(conn, 400, "unable to process records\n"); - } - - return ret; -} - - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} - /* New gen HTTP server */ static int send_response_ng(struct flb_http_response *response, @@ -1247,7 +533,7 @@ static int process_pack_ng(struct flb_http *ctx, flb_sds_t tag, char *buf, size_ flb_time_get(&tm); if (ctx->add_remote_addr == FLB_TRUE && ctx->remote_addr_key != NULL) { - remote_addr = get_remote_addr(request, HTTP_PROTOCOL_VERSION_20); + remote_addr = get_remote_addr((struct flb_http_request *) request); } msgpack_unpacked_init(&result); @@ -1443,11 +729,11 @@ static int process_payload_ng(flb_sds_t tag, return -1; } - if (strcasecmp(request->content_type, "application/json") == 0) { + if (content_type_is_json(request->content_type)) { type = HTTP_CONTENT_JSON; } - if (strcasecmp(request->content_type, "application/x-www-form-urlencoded") == 0) { + if (content_type_is_urlencoded(request->content_type)) { type = HTTP_CONTENT_URLENCODED; } @@ -1469,7 +755,8 @@ static int process_payload_ng(flb_sds_t tag, ctx = (struct flb_http *) request->stream->user_data; payload = (char *) request->body; if (payload) { - return parse_payload_urlencoded(ctx, tag, payload, cfl_sds_len(payload), request); + return parse_payload_urlencoded(ctx, tag, payload, + cfl_sds_len(payload), request); } } @@ -1521,7 +808,7 @@ int http_prot_handle_ng(struct flb_http_request *request, if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { flb_sds_destroy(tag); - + send_response_ng(response, 400, "error: missing host header\n"); return -1; } diff --git a/plugins/in_http/http_prot.h b/plugins/in_http/http_prot.h index 8b48841c358..7d5f5179f84 100644 --- a/plugins/in_http/http_prot.h +++ b/plugins/in_http/http_prot.h @@ -22,15 +22,7 @@ #include -int http_prot_handle(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - int http_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response); -int http_prot_handle_error(struct flb_http *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - #endif From cd2ba1db0027c12cf15ea8007390c6555e4e4e43 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:24:10 -0600 Subject: [PATCH 05/42] in_prometheus_remote_write: unify HTTP server path Signed-off-by: Eduardo Silva --- .../in_prometheus_remote_write/CMakeLists.txt | 1 - plugins/in_prometheus_remote_write/prom_rw.c | 159 ++----- plugins/in_prometheus_remote_write/prom_rw.h | 13 - .../prom_rw_config.c | 37 +- .../in_prometheus_remote_write/prom_rw_conn.c | 327 ------------- .../in_prometheus_remote_write/prom_rw_conn.h | 57 --- .../in_prometheus_remote_write/prom_rw_prot.c | 438 ++---------------- .../in_prometheus_remote_write/prom_rw_prot.h | 11 - 8 files changed, 56 insertions(+), 987 deletions(-) delete mode 100644 plugins/in_prometheus_remote_write/prom_rw_conn.c delete mode 100644 plugins/in_prometheus_remote_write/prom_rw_conn.h diff --git a/plugins/in_prometheus_remote_write/CMakeLists.txt b/plugins/in_prometheus_remote_write/CMakeLists.txt index ee8ce703ae2..0e584c3b962 100644 --- a/plugins/in_prometheus_remote_write/CMakeLists.txt +++ b/plugins/in_prometheus_remote_write/CMakeLists.txt @@ -5,7 +5,6 @@ endif() set(src prom_rw.c prom_rw_prot.c - prom_rw_conn.c prom_rw_config.c ) diff --git a/plugins/in_prometheus_remote_write/prom_rw.c b/plugins/in_prometheus_remote_write/prom_rw.c index a7f0d5d31b8..df96f17a2fa 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.c +++ b/plugins/in_prometheus_remote_write/prom_rw.c @@ -19,54 +19,18 @@ #include -#include -#include #include #include "prom_rw.h" -#include "prom_rw_conn.h" #include "prom_rw_prot.h" #include "prom_rw_config.h" -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int prom_rw_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct prom_remote_write_conn *conn; - struct flb_prom_remote_write *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); - - conn = prom_rw_conn_add(connection, ctx); - - if (conn == NULL) { - return -1; - } - - return 0; -} - static int prom_rw_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_prom_remote_write *ctx; + struct flb_http_server_options http_server_options; (void) data; @@ -75,7 +39,6 @@ static int prom_rw_init(struct flb_input_instance *ins, if (!ctx) { return -1; } - ctx->collector_id = -1; /* Populate context with config map defaults and incoming properties */ ret = flb_input_config_map_set(ins, (void *) ctx); @@ -88,86 +51,40 @@ static int prom_rw_init(struct flb_input_instance *ins, /* Set the context */ flb_input_set_context(ins, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_VERSION_AUTODETECT, - (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), - NULL, - ins->host.listen, - ins->host.port, - ins->tls, - ins->flags, - &ins->net_setup, - flb_input_event_loop_get(ins), - ins->config, - (void *) ctx); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not initialize http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - prom_rw_config_destroy(ctx); - - return -1; + ret = flb_input_http_server_options_init(&http_server_options, + ins, + (FLB_HTTP_SERVER_FLAG_KEEPALIVE | + FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), + prom_rw_prot_handle_ng, + ctx); + if (ret == 0) { + ret = flb_http_server_init_with_options(&ctx->http_server, + &http_server_options); + + if (ret == 0) { + ret = flb_http_server_start(&ctx->http_server); } - ret = flb_http_server_start(&ctx->http_server); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not start http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - prom_rw_config_destroy(ctx); - - return -1; + if (ret == 0) { + ret = flb_input_downstream_set(ctx->http_server.downstream, ins); } - - flb_http_server_set_buffer_max_size(&ctx->http_server, ctx->buffer_max_size); - - ctx->http_server.request_callback = prom_rw_prot_handle_ng; - - flb_input_downstream_set(ctx->http_server.downstream, ctx->ins); } - else { - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - prom_rw_config_destroy(ctx); - - return -1; - } - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - prom_rw_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - prom_rw_config_destroy(ctx); - return -1; - } + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start http server on %s:%u. Aborting", + ins->host.listen, ins->host.port); - ctx->collector_id = ret; + prom_rw_config_destroy(ctx); + return -1; } - flb_plg_info(ctx->ins, "listening on %s:%s", ctx->listen, ctx->tcp_port); + flb_plg_info(ctx->ins, + "listening on %s:%u with %i worker%s", + ins->host.listen, + ins->host.port, + ctx->http_server.workers, + ctx->http_server.workers == 1 ? "" : "s"); if (ctx->successful_response_code != 200 && ctx->successful_response_code != 201 && @@ -197,24 +114,6 @@ static int prom_rw_exit(void *data, struct flb_config *config) /* Configuration properties map */ static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "http2", "true", - 0, FLB_TRUE, offsetof(struct flb_prom_remote_write, enable_http2), - "Enable HTTP/2 support" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_prom_remote_write, buffer_max_size), - "Maximum size of the HTTP request buffer" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_prom_remote_write, buffer_chunk_size), - "Size of each buffer chunk allocated for HTTP requests" - }, - { FLB_CONFIG_MAP_STR, "uri", NULL, 0, FLB_TRUE, offsetof(struct flb_prom_remote_write, uri), @@ -242,11 +141,11 @@ struct flb_input_plugin in_prometheus_remote_write_plugin = { .description = "Prometheus Remote Write input", .cb_init = prom_rw_init, .cb_pre_run = NULL, - .cb_collect = prom_rw_collect, + .cb_collect = NULL, .cb_flush_buf = NULL, .cb_pause = NULL, .cb_resume = NULL, .cb_exit = prom_rw_exit, .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS + .flags = FLB_INPUT_NET_SERVER | FLB_INPUT_HTTP_SERVER | FLB_IO_OPT_TLS }; diff --git a/plugins/in_prometheus_remote_write/prom_rw.h b/plugins/in_prometheus_remote_write/prom_rw.h index 32c90d5a6a6..9a1bd05e2f8 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.h +++ b/plugins/in_prometheus_remote_write/prom_rw.h @@ -24,7 +24,6 @@ #include #include -#include #include #define HTTP_BUFFER_MAX_SIZE "4M" @@ -41,19 +40,7 @@ struct flb_prom_remote_write { /* HTTP URI */ char *uri; - /* New gen HTTP server */ - int enable_http2; struct flb_http_server http_server; - - /* Legacy HTTP server */ - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; }; diff --git a/plugins/in_prometheus_remote_write/prom_rw_config.c b/plugins/in_prometheus_remote_write/prom_rw_config.c index 7804595dc9c..c6474bfa93d 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_config.c +++ b/plugins/in_prometheus_remote_write/prom_rw_config.c @@ -18,11 +18,9 @@ */ #include -#include #include "prom_rw.h" #include "prom_rw_config.h" -#include "prom_rw_conn.h" /* default HTTP port for prometheus remote write */ #define PROMETHEUS_REMOTE_WRITE_HTTP_PORT 8080 @@ -39,7 +37,6 @@ struct flb_prom_remote_write *prom_rw_config_create(struct flb_input_instance *i return NULL; } ctx->ins = ins; - mk_list_init(&ctx->connections); /* Load the config map */ ret = flb_input_config_map_set(ins, (void *) ctx); @@ -55,44 +52,12 @@ struct flb_prom_remote_write *prom_rw_config_create(struct flb_input_instance *i snprintf(port, sizeof(port) - 1, "%d", ins->host.port); ctx->tcp_port = flb_strdup(port); - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - if (ctx->server == NULL) { - flb_plg_error(ctx->ins, "error on mk_server allocation"); - prom_rw_config_destroy(ctx); - return NULL; - } - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - return ctx; } int prom_rw_config_destroy(struct flb_prom_remote_write *ctx) { - /* release all connections */ - prom_rw_conn_release_all(ctx); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->enable_http2) { - flb_http_server_destroy(&ctx->http_server); - } - - if (ctx->server) { - flb_free(ctx->server); - } + flb_http_server_destroy(&ctx->http_server); flb_free(ctx->listen); flb_free(ctx->tcp_port); diff --git a/plugins/in_prometheus_remote_write/prom_rw_conn.c b/plugins/in_prometheus_remote_write/prom_rw_conn.c deleted file mode 100644 index 4a32c423851..00000000000 --- a/plugins/in_prometheus_remote_write/prom_rw_conn.c +++ /dev/null @@ -1,327 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "prom_rw.h" -#include "prom_rw_conn.h" -#include "prom_rw_prot.h" - -static void prom_rw_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - - -static int prom_rw_conn_buffer_realloc(struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, size_t size) -{ - char *tmp; - - /* Perform realloc */ - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - flb_plg_error(ctx->ins, "could not perform realloc for size %zu", size); - return -1; - } - - /* Update buffer info */ - conn->buf_data = tmp; - conn->buf_size = size; - - /* Keep NULL termination */ - conn->buf_data[conn->buf_len] = '\0'; - - /* Reset parser state */ - mk_http_parser_init(&conn->session.parser); - - return 0; -} - -static int prom_rw_conn_event(void *data) -{ - int ret; - int status; - size_t size; - ssize_t available; - ssize_t bytes; - char *request_end; - size_t request_len; - struct prom_remote_write_conn *conn; - struct mk_event *event; - struct flb_prom_remote_write *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - prom_rw_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - ret = prom_rw_conn_buffer_realloc(ctx, conn, size); - if (ret == -1) { - flb_errno(); - prom_rw_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - prom_rw_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - prom_rw_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - request_end = NULL; - - if (NULL != conn->request.data.data) { - request_end = &conn->request.data.data[conn->request.data.len]; - } - else { - request_end = strstr(conn->buf_data, "\r\n\r\n"); - - if(NULL != request_end) { - request_end = &request_end[4]; - } - } - - if (NULL != request_end) { - request_len = (size_t)(request_end - conn->buf_data); - - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - prom_rw_conn_request_init(&conn->session, &conn->request); - } - } - else if (status == MK_HTTP_PARSER_ERROR) { - prom_rw_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - prom_rw_conn_request_init(&conn->session, &conn->request); - } - - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - prom_rw_conn_del(conn); - return -1; - } - - return 0; - -} - -static void prom_rw_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void prom_rw_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct prom_remote_write_conn *prom_rw_conn_add(struct flb_connection *connection, - struct flb_prom_remote_write *ctx) -{ - struct prom_remote_write_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct prom_remote_write_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = prom_rw_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - flb_free(conn->buf_data); - flb_free(conn); - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - prom_rw_conn_session_init(&conn->session, ctx->server, connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - prom_rw_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - return conn; -} - -int prom_rw_conn_del(struct prom_remote_write_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void prom_rw_conn_release_all(struct flb_prom_remote_write *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct prom_remote_write_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct prom_remote_write_conn, _head); - prom_rw_conn_del(conn); - } -} diff --git a/plugins/in_prometheus_remote_write/prom_rw_conn.h b/plugins/in_prometheus_remote_write/prom_rw_conn.h deleted file mode 100644 index cfe5849002f..00000000000 --- a/plugins/in_prometheus_remote_write/prom_rw_conn.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_PROM_RW_CONN_H -#define FLB_IN_PROM_RW_CONN_H - -#include -#include -#include -#include - -#include "prom_rw_conn.h" - -struct prom_remote_write_conn { - struct mk_event event; /* Built-in event data for mk_events */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_opentelemetry->connections */ -}; - -struct prom_remote_write_conn *prom_rw_conn_add(struct flb_connection *connection, - struct flb_prom_remote_write *ctx); -int prom_rw_conn_del(struct prom_remote_write_conn *conn); -void prom_rw_conn_release_all(struct flb_prom_remote_write *ctx); - - -#endif diff --git a/plugins/in_prometheus_remote_write/prom_rw_prot.c b/plugins/in_prometheus_remote_write/prom_rw_prot.c index abb9275fd0b..96e4f5543b3 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_prot.c +++ b/plugins/in_prometheus_remote_write/prom_rw_prot.c @@ -18,395 +18,12 @@ */ #include -#include -#include -#include -#include -#include -#include - -#include #include -#include #include #include "prom_rw.h" -#include "prom_rw_conn.h" - -static int send_response(struct flb_input_instance *in, - struct prom_remote_write_conn *conn, - int http_status, char *message) -{ - int len; - flb_sds_t out; - size_t sent; - ssize_t bytes; - int result; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "\r\n", - FLB_VERSION_STR); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check the outcome of this operation */ - bytes = flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - if (bytes == -1) { - flb_plg_error(in, "cannot send response"); - - result = -1; - } - else { - result = 0; - } - - flb_sds_destroy(out); - - return result; -} - -static int process_payload_metrics(struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request) -{ - struct cmt *context; - int result; - - result = cmt_decode_prometheus_remote_write_create(&context, - request->data.data, - request->data.len); - - if (result == CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - result = flb_input_metrics_append(ctx->ins, NULL, 0, context); - - cmt_decode_prometheus_remote_write_destroy(context); - if (result != 0) { - flb_plg_debug(ctx->ins, "could not ingest metrics : %d", result); - return -1; - } - - return 0; - } - - return -1; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -static int uncompress_snappy(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_snappy_uncompress_framed_data(input_buffer, - input_size, - output_buffer, - output_size); - - if (ret != 0) { - flb_error("[opentelemetry] snappy decompression failed"); - - return -1; - } - - return 1; -} - -static int uncompress_gzip(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_gzip_uncompress(input_buffer, - input_size, - (void *) output_buffer, - output_size); - - if (ret == -1) { - flb_error("[opentelemetry] gzip decompression failed"); - - return -1; - } - - return 1; -} - -int prom_rw_prot_uncompress(struct mk_http_session *session, - struct mk_http_request *request, - char **output_buffer, - size_t *output_size) -{ - struct mk_http_header *header; - size_t index; - - *output_buffer = NULL; - *output_size = 0; - - for (index = 0; - index < session->parser.headers_extra_count; - index++) { - header = &session->parser.headers_extra[index]; - - if (strncasecmp(header->key.data, "Content-Encoding", 16) == 0) { - if (strncasecmp(header->val.data, "gzip", 4) == 0) { - return uncompress_gzip(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else if (strncasecmp(header->val.data, "snappy", 6) == 0) { - return uncompress_snappy(output_buffer, - output_size, - request->data.data, - request->data.len); - } - else { - return -2; - } - } - } - - return 0; -} - -/* - * Handle an incoming request. It performs extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int prom_rw_prot_handle(struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret = -1; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - char *original_data; - size_t original_data_size; - char *uncompressed_data; - size_t uncompressed_data_size; - - if (request->uri.data[0] != '/') { - send_response(ctx->ins, conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - if (ctx->uri != NULL && strcmp(uri, ctx->uri) != 0) { - send_response(ctx->ins, conn, 400, "error: invalid endpoint\n"); - mk_mem_free(uri); - - return -1; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Compose the query string using the URI */ - len = strlen(uri); - - if (ctx->tag_from_uri != FLB_TRUE) { - tag = flb_sds_create(ctx->ins->tag); - } - else { - tag = flb_sds_create_size(len); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* New tag skipping the URI '/' */ - flb_sds_cat_safe(&tag, uri + 1, len - 1); - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - mk_http_point_header(&request->content_type, &session->parser, MK_HEADER_CONTENT_TYPE); - - if (request->method != MK_METHOD_POST) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(ctx->ins, conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - if (request->data.data == NULL || request->data.len <= 0) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(ctx->ins, conn, 400, "error: no payload found\n"); - return -1; - } - - original_data = request->data.data; - original_data_size = request->data.len; - - ret = prom_rw_prot_uncompress(session, request, - &uncompressed_data, - &uncompressed_data_size); - - if (ret > 0) { - request->data.data = uncompressed_data; - request->data.len = uncompressed_data_size; - } - - if (ctx->uri != NULL && strcmp(uri, ctx->uri) == 0) { - ret = process_payload_metrics(ctx, conn, tag, session, request); - } - else { - ret = process_payload_metrics(ctx, conn, tag, session, request); - } - - if (uncompressed_data != NULL) { - flb_free(uncompressed_data); - } - - request->data.data = original_data; - request->data.len = original_data_size; - - mk_mem_free(uri); - flb_sds_destroy(tag); - - if (ret == -1) { - send_response(ctx->ins, conn, 400, "error: invalid request\n"); - return -1; - } - - send_response(ctx->ins, conn, ctx->successful_response_code, NULL); - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int prom_rw_prot_handle_error( - struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(ctx->ins, conn, 400, "error: invalid request\n"); - return -1; -} - - -/* New gen HTTP server */ static int send_response_ng(struct flb_http_response *response, int http_status, char *message) @@ -438,9 +55,7 @@ static int send_response_ng(struct flb_http_response *response, } static int process_payload_metrics_ng(struct flb_prom_remote_write *ctx, - flb_sds_t tag, - struct flb_http_request *request, - struct flb_http_response *response) + struct flb_http_request *request) { struct cmt *context; int result; @@ -449,14 +64,16 @@ static int process_payload_metrics_ng(struct flb_prom_remote_write *ctx, request->body, cfl_sds_len(request->body)); - if (result == CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - result = flb_input_metrics_append(ctx->ins, NULL, 0, context); + if (result != CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + return -1; + } - if (result != 0) { - flb_plg_debug(ctx->ins, "could not ingest metrics : %d", result); - } + result = flb_input_metrics_append(ctx->ins, NULL, 0, context); + cmt_decode_prometheus_remote_write_destroy(context); - cmt_decode_prometheus_remote_write_destroy(context); + if (result != 0) { + flb_plg_debug(ctx->ins, "could not ingest metrics : %d", result); + return -1; } return 0; @@ -465,47 +82,44 @@ static int process_payload_metrics_ng(struct flb_prom_remote_write *ctx, int prom_rw_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response) { - struct flb_prom_remote_write *context; + struct flb_prom_remote_write *ctx; int result; - context = (struct flb_prom_remote_write *) response->stream->user_data; + ctx = (struct flb_prom_remote_write *) response->stream->user_data; - if (request->path[0] != '/') { + if (request->path == NULL || request->path[0] != '/') { send_response_ng(response, 400, "error: invalid request\n"); return -1; } - /* ToDo: Fix me */ - /* HTTP/1.1 needs Host header */ - if (request->protocol_version >= HTTP_PROTOCOL_VERSION_11 && - request->host == NULL) { + if (ctx->uri != NULL && strcmp(request->path, ctx->uri) != 0) { + send_response_ng(response, 400, "error: invalid endpoint\n"); return -1; } - if (request->method != HTTP_METHOD_POST) { - send_response_ng(response, 400, "error: invalid HTTP method\n"); + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && + request->host == NULL) { + send_response_ng(response, 400, "error: missing host header\n"); return -1; } - /* check content-length */ - if (request->content_length <= 0) { - send_response_ng(response, 400, "error: invalid content-length\n"); + if (request->method != HTTP_METHOD_POST) { + send_response_ng(response, 400, "error: invalid HTTP method\n"); return -1; } - if (request->body == NULL) { + if (request->body == NULL || cfl_sds_len(request->body) == 0) { send_response_ng(response, 400, "error: invalid payload\n"); return -1; } - if (context->uri != NULL && strcmp(request->path, context->uri) == 0) { - result = process_payload_metrics_ng(context, context->ins->tag, request, response); - } - else { - result = process_payload_metrics_ng(context, context->ins->tag, request, response); + result = process_payload_metrics_ng(ctx, request); + if (result != 0) { + send_response_ng(response, 400, "error: invalid request\n"); + return -1; } - send_response_ng(response, context->successful_response_code, NULL); + send_response_ng(response, ctx->successful_response_code, NULL); - return result; + return 0; } diff --git a/plugins/in_prometheus_remote_write/prom_rw_prot.h b/plugins/in_prometheus_remote_write/prom_rw_prot.h index a8548b37485..74d3fa907eb 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_prot.h +++ b/plugins/in_prometheus_remote_write/prom_rw_prot.h @@ -22,17 +22,6 @@ #include -int prom_rw_prot_handle(struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int prom_rw_prot_handle_error(struct flb_prom_remote_write *ctx, - struct prom_remote_write_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - - int prom_rw_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response); From d4d82edfdcb81dcca06298af4c508b1cc5617032 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:24:29 -0600 Subject: [PATCH 06/42] in_splunk: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/in_splunk/CMakeLists.txt | 1 - plugins/in_splunk/splunk.c | 193 ++------ plugins/in_splunk/splunk.h | 11 - plugins/in_splunk/splunk_config.c | 37 +- plugins/in_splunk/splunk_conn.c | 337 -------------- plugins/in_splunk/splunk_conn.h | 54 --- plugins/in_splunk/splunk_prot.c | 701 +----------------------------- plugins/in_splunk/splunk_prot.h | 8 - 8 files changed, 46 insertions(+), 1296 deletions(-) delete mode 100644 plugins/in_splunk/splunk_conn.c delete mode 100644 plugins/in_splunk/splunk_conn.h diff --git a/plugins/in_splunk/CMakeLists.txt b/plugins/in_splunk/CMakeLists.txt index 42ecf2e31a6..9fef59d26b8 100644 --- a/plugins/in_splunk/CMakeLists.txt +++ b/plugins/in_splunk/CMakeLists.txt @@ -4,7 +4,6 @@ endif() set(src splunk.c - splunk_conn.c splunk_prot.c splunk_config.c ) diff --git a/plugins/in_splunk/splunk.c b/plugins/in_splunk/splunk.c index 13a18ce4013..adb67045f16 100644 --- a/plugins/in_splunk/splunk.c +++ b/plugins/in_splunk/splunk.c @@ -23,53 +23,17 @@ #include #include "splunk.h" -#include "splunk_conn.h" #include "splunk_prot.h" #include "splunk_config.h" -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_splunk_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct splunk_conn *conn; - struct flb_splunk *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = splunk_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} - static int in_splunk_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; - int ret; - struct flb_splunk *ctx; + int ret; + struct flb_splunk *ctx; + struct flb_http_server_options http_server_options; + (void) config; (void) data; /* Create context and basic conf */ @@ -78,8 +42,6 @@ static int in_splunk_init(struct flb_input_instance *ins, return -1; } - ctx->collector_id = -1; - /* Populate context with config map defaults and incoming properties */ ret = flb_input_config_map_set(ins, (void *) ctx); if (ret == -1) { @@ -91,93 +53,38 @@ static int in_splunk_init(struct flb_input_instance *ins, /* Set the context */ flb_input_set_context(ins, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - - if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_VERSION_AUTODETECT, - (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), - NULL, - ins->host.listen, - ins->host.port, - ins->tls, - ins->flags, - &ins->net_setup, - flb_input_event_loop_get(ins), - ins->config, - (void *) ctx); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not initialize http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - splunk_config_destroy(ctx); - - return -1; + ret = flb_input_http_server_options_init( + &http_server_options, + ins, + (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), + splunk_prot_handle_ng, + ctx); + if (ret == 0) { + ret = flb_http_server_init_with_options(&ctx->http_server, + &http_server_options); + + if (ret == 0) { + ret = flb_http_server_start(&ctx->http_server); } - ret = flb_http_server_start(&ctx->http_server); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not start http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - splunk_config_destroy(ctx); - - return -1; + if (ret == 0) { + ret = flb_input_downstream_set(ctx->http_server.downstream, ins); } - - flb_http_server_set_buffer_max_size(&ctx->http_server, ctx->buffer_max_size); - - ctx->http_server.request_callback = splunk_prot_handle_ng; - - flb_input_downstream_set(ctx->http_server.downstream, ctx->ins); - - flb_plg_info(ctx->ins, "listening on %s:%u", - ins->host.listen, ins->host.port); } - else { - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - splunk_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - flb_plg_info(ctx->ins, "listening on %s:%s", ctx->listen, ctx->tcp_port); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_splunk_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - splunk_config_destroy(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start http server on %s:%u. Aborting", + ins->host.listen, ins->host.port); - return -1; - } - - ctx->collector_id = ret; + splunk_config_destroy(ctx); + return -1; } - + flb_plg_info(ctx->ins, "listening on %s:%u with %i worker%s", + ins->host.listen, + ins->host.port, + ctx->http_server.workers, + ctx->http_server.workers == 1 ? "" : "s"); return 0; } @@ -196,42 +103,8 @@ static int in_splunk_exit(void *data, struct flb_config *config) return 0; } - -static void in_splunk_pause(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx = data; - - flb_input_collector_pause(ctx->collector_id, ctx->ins); - -} - -static void in_splunk_resume(void *data, struct flb_config *config) -{ - struct flb_splunk *ctx = data; - - flb_input_collector_resume(ctx->collector_id, ctx->ins); -} - /* Configuration properties map */ static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "http2", "true", - 0, FLB_TRUE, offsetof(struct flb_splunk, enable_http2), - "Enable HTTP/2 support" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_splunk, buffer_max_size), - "Set the maximum size of buffer" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_splunk, buffer_chunk_size), - "Set the initial buffer size to store incoming data" - }, - { FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_splunk, success_headers), @@ -283,11 +156,11 @@ struct flb_input_plugin in_splunk_plugin = { .description = "Input plugin for Splunk HEC payloads", .cb_init = in_splunk_init, .cb_pre_run = NULL, - .cb_collect = in_splunk_collect, + .cb_collect = NULL, .cb_flush_buf = NULL, - .cb_pause = in_splunk_pause, - .cb_resume = in_splunk_resume, + .cb_pause = NULL, + .cb_resume = NULL, .cb_exit = in_splunk_exit, .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS + .flags = FLB_INPUT_NET_SERVER | FLB_INPUT_HTTP_SERVER | FLB_IO_OPT_TLS }; diff --git a/plugins/in_splunk/splunk.h b/plugins/in_splunk/splunk.h index 93db1f444bf..c8265fd527c 100644 --- a/plugins/in_splunk/splunk.h +++ b/plugins/in_splunk/splunk.h @@ -20,14 +20,12 @@ #ifndef FLB_IN_SPLUNK_H #define FLB_IN_SPLUNK_H -#include #include #include #include #include #include -#include #include #define HTTP_BUFFER_MAX_SIZE "4M" @@ -61,18 +59,9 @@ struct flb_splunk { struct flb_input_instance *ins; - /* New gen HTTP server */ - int enable_http2; struct flb_http_server http_server; - /* Legacy HTTP server */ flb_sds_t success_headers_str; - int collector_id; - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - struct mk_server *server; }; diff --git a/plugins/in_splunk/splunk_config.c b/plugins/in_splunk/splunk_config.c index 7e3d058f208..14554e53f3d 100644 --- a/plugins/in_splunk/splunk_config.c +++ b/plugins/in_splunk/splunk_config.c @@ -21,7 +21,6 @@ #include "splunk.h" #include "splunk_config.h" -#include "splunk_conn.h" #include "splunk_config.h" static void delete_hec_tokens(struct flb_splunk *ctx) @@ -134,7 +133,6 @@ struct flb_splunk *splunk_config_create(struct flb_input_instance *ins) return NULL; } ctx->ins = ins; - mk_list_init(&ctx->connections); mk_list_init(&ctx->auth_tokens); /* Load the config map */ @@ -159,19 +157,6 @@ struct flb_splunk *splunk_config_create(struct flb_input_instance *ins) snprintf(port, sizeof(port) - 1, "%d", ins->host.port); ctx->tcp_port = flb_strdup(port); - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - if (ctx->server == NULL) { - flb_plg_error(ctx->ins, "error on mk_server allocation"); - splunk_config_destroy(ctx); - return NULL; - } - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - ret = flb_log_event_encoder_init(&ctx->log_encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); @@ -248,28 +233,8 @@ int splunk_config_destroy(struct flb_splunk *ctx) flb_ra_destroy(ctx->ra_tag_key); } - /* release all connections */ - splunk_conn_release_all(ctx); - flb_log_event_encoder_destroy(&ctx->log_encoder); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->enable_http2) { - flb_http_server_destroy(&ctx->http_server); - } - - if (ctx->server) { - flb_free(ctx->server); - } + flb_http_server_destroy(&ctx->http_server); if (ctx->success_headers_str != NULL) { flb_sds_destroy(ctx->success_headers_str); diff --git a/plugins/in_splunk/splunk_conn.c b/plugins/in_splunk/splunk_conn.c deleted file mode 100644 index b1a3a09d2fa..00000000000 --- a/plugins/in_splunk/splunk_conn.c +++ /dev/null @@ -1,337 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for thet specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "splunk.h" -#include "splunk_conn.h" -#include "splunk_prot.h" - -static void splunk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static void splunk_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd); - -static int splunk_conn_buffer_realloc(struct flb_splunk *ctx, - struct splunk_conn *conn, - size_t size) -{ - char *tmp; - - flb_plg_trace(ctx->ins, "realloc buffer %i -> %zu bytes", - (int)conn->buf_size, size); - - /* Perform realloc */ - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - flb_plg_error(ctx->ins, "could not perform realloc for size %zu", size); - return -1; - } - - /* Update buffer info */ - conn->buf_data = tmp; - conn->buf_size = size; - /* Keep NULL termination */ - conn->buf_data[conn->buf_len] = '\0'; - - /* Reset parser state */ - mk_http_parser_init(&conn->session.parser); - - flb_plg_trace(ctx->ins, "realloc completed successfully size=%zu", size); - return 0; -} - - -static int splunk_conn_event(void *data) -{ - int ret; - int status; - size_t size; - ssize_t available; - ssize_t bytes; - size_t request_len; - struct flb_connection *connection; - struct splunk_conn *conn; - struct mk_event *event; - struct flb_splunk *ctx; - - connection = (struct flb_connection *) data; - conn = connection->user_data; - ctx = conn->ctx; - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - splunk_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - ret = splunk_conn_buffer_realloc(ctx, conn, size); - if (ret == -1) { - flb_errno(); - splunk_conn_del(conn); - return -1; - } - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - splunk_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - ret = splunk_prot_handle(ctx, conn, &conn->session, &conn->request); - if (ret == -1) { - splunk_conn_del(conn); - return -1; - } - - /* - * Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - /* Use the last parser position as the request length */ - request_len = mk_http_parser_request_size(&conn->session.parser, - conn->buf_data, - conn->buf_len); - - if (request_len == -1 || (request_len > conn->buf_len)) { - /* Unexpected but let's make sure things are safe */ - conn->buf_len = 0; - flb_plg_debug(ctx->ins, "request length exceeds buffer length, closing connection"); - splunk_conn_del(conn); - return -1; - } - - /* If we have extra bytes in our bytes, adjust the extra bytes */ - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - splunk_conn_request_init(&conn->session, &conn->request); - } - else if (status == MK_HTTP_PARSER_ERROR) { - splunk_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - splunk_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - splunk_conn_del(conn); - return -1; - } - - return 0; -} - - -static void splunk_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void splunk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct splunk_conn *splunk_conn_add(struct flb_connection *connection, - struct flb_splunk *ctx) -{ - struct splunk_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct splunk_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = splunk_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - splunk_conn_session_init(&conn->session, ctx->server, conn->connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - splunk_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int splunk_conn_del(struct splunk_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void splunk_conn_release_all(struct flb_splunk *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct splunk_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct splunk_conn, _head); - splunk_conn_del(conn); - } -} \ No newline at end of file diff --git a/plugins/in_splunk/splunk_conn.h b/plugins/in_splunk/splunk_conn.h deleted file mode 100644 index 59eeb24fd21..00000000000 --- a/plugins/in_splunk/splunk_conn.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_SPLUNK_CONN -#define FLB_IN_SPLUNK_CONN - -#include -#include - -#include -#include -#include - -struct splunk_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_http->connections */ -}; - -struct splunk_conn *splunk_conn_add(struct flb_connection *connection, struct flb_splunk *ctx); -int splunk_conn_del(struct splunk_conn *conn); -void splunk_conn_release_all(struct flb_splunk *ctx); - - -#endif diff --git a/plugins/in_splunk/splunk_prot.c b/plugins/in_splunk/splunk_prot.c index 0d81ef26304..f2048fdbde8 100644 --- a/plugins/in_splunk/splunk_prot.c +++ b/plugins/in_splunk/splunk_prot.c @@ -18,7 +18,6 @@ */ #include -#include #include #include #include @@ -29,203 +28,13 @@ #include #include -#include -#include - #include "splunk.h" -#include "splunk_conn.h" #include "splunk_prot.h" #define HTTP_CONTENT_JSON 0 #define HTTP_CONTENT_TEXT 1 #define HTTP_CONTENT_UNKNOWN 2 -static int send_response(struct splunk_conn *conn, int http_status, char *message) -{ - struct flb_splunk *context; - size_t sent; - int len; - flb_sds_t out; - - context = (struct flb_splunk *) conn->ctx; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "%s" - "\r\n\r\n", - FLB_VERSION_STR, - context->success_headers_str); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - else if (http_status == 401) { - flb_sds_printf(&out, - "HTTP/1.1 401 Unauthorized\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_json_message_response(struct splunk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, message); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -/* - * We use two backends for HTTP parsing and it depends on the version of the - * protocol: - * - * http/1.x: we use Monkey HTTP parser: struct mk_http_session.parser - */ -static int http_header_lookup(int version, void *ptr, char *key, - char **val, size_t *val_len) -{ - int key_len; - - /* HTTP/1.1 */ - struct mk_list *head; - struct mk_http_session *session; - struct mk_http_request *request_11; - struct mk_http_header *header; - - /* HTTP/2.0 */ - char *value; - struct flb_http_request *request_20; - - if (!key) { - return -1; - } - - key_len = strlen(key); - if (key_len <= 0) { - return -1; - } - - if (version <= HTTP_PROTOCOL_VERSION_11) { - if (!ptr) { - return -1; - } - - request_11 = (struct mk_http_request *) ptr; - session = request_11->session; - mk_list_foreach(head, &session->parser.header_list) { - header = mk_list_entry(head, struct mk_http_header, _head); - if (header->key.len == key_len && - strncasecmp(header->key.data, key, key_len) == 0) { - *val = header->val.data; - *val_len = header->val.len; - return 0; - } - } - return -1; - } - else if (version == HTTP_PROTOCOL_VERSION_20) { - request_20 = ptr; - if (!request_20) { - return -1; - } - - value = flb_http_request_get_header(request_20, key); - if (!value) { - return -1; - } - - *val = value; - *val_len = strlen(value); - return 0; - } - - return -1; -} - static void extract_xff_value(const char *value, size_t value_len, const char **out_value, size_t *out_len) { @@ -265,7 +74,7 @@ static void extract_xff_value(const char *value, size_t value_len, static int extract_remote_address(const char *xff_value, size_t xff_value_len, struct flb_connection *connection, - char **out, + const char **out, size_t *out_len) { const char *value = NULL; @@ -630,57 +439,6 @@ static ssize_t parse_hec_payload_json(struct flb_splunk *ctx, flb_sds_t tag, return 0; } -static int validate_auth_header(struct flb_splunk *ctx, struct mk_http_request *request) -{ - int ret = 0; - struct mk_list *head; - struct mk_http_header *auth_header = NULL; - struct flb_splunk_tokens *splunk_token; - flb_sds_t authorization = NULL; - - if (mk_list_size(&ctx->auth_tokens) == 0) { - return SPLUNK_AUTH_UNAUTH; - } - - auth_header = mk_http_header_get(MK_HEADER_AUTHORIZATION, request, NULL, 0); - if (auth_header == NULL) { - return SPLUNK_AUTH_MISSING_CRED; - } - - authorization = flb_sds_create_len(auth_header->val.data, auth_header->val.len); - if (authorization == NULL) { - return SPLUNK_AUTH_MISSING_CRED; - } - - if (authorization != NULL && auth_header->val.len > 0) { - mk_list_foreach(head, &ctx->auth_tokens) { - splunk_token = mk_list_entry(head, struct flb_splunk_tokens, _head); - if (flb_sds_len(authorization) != splunk_token->length) { - ret = SPLUNK_AUTH_UNAUTHORIZED; - continue; - } - - if (strncasecmp(splunk_token->header, - authorization, - splunk_token->length) == 0) { - flb_sds_destroy(authorization); - - return SPLUNK_AUTH_SUCCESS; - } - } - - ret = SPLUNK_AUTH_UNAUTHORIZED; - flb_sds_destroy(authorization); - return ret; - } - else { - flb_sds_destroy(authorization); - return SPLUNK_AUTH_MISSING_CRED; - } - - return SPLUNK_AUTH_SUCCESS; -} - static int handle_hec_payload(struct flb_splunk *ctx, int content_type, flb_sds_t tag, char *buf, size_t size, const char *remote_addr, @@ -710,449 +468,6 @@ static int handle_hec_payload(struct flb_splunk *ctx, int content_type, return ret; } -static int process_hec_payload(struct flb_splunk *ctx, struct splunk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request, - const char *remote_addr, - size_t remote_addr_len) -{ - int i = 0; - int ret = 0; - int type = -1; - struct mk_http_header *header; - struct mk_http_header *header_auth; - int extra_size = -1; - struct mk_http_header *headers_extra; - int gzip_compressed = FLB_FALSE; - void *gz_data = NULL; - size_t gz_size = -1; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - flb_plg_debug(ctx->ins, "header 'Content-Type' is not set"); - } - - if (header->val.len == 16 && - strncasecmp(header->val.data, "application/json", 16) == 0) { - type = HTTP_CONTENT_JSON; - } - else if (header->val.len == 10 && - strncasecmp(header->val.data, "text/plain", 10) == 0) { - type = HTTP_CONTENT_TEXT; - } - else { - /* Not necessary to specify content-type for Splunk HEC. */ - flb_plg_debug(ctx->ins, "Mark as unknown type for ingested payloads"); - type = HTTP_CONTENT_UNKNOWN; - } - - if (request->data.len <= 0 && !mk_http_parser_is_content_chunked(&session->parser)) { - send_json_message_response(conn, 400, "{\"text\":\"No data\",\"code\":5}"); - return -2; - } - - header_auth = &session->parser.headers[MK_HEADER_AUTHORIZATION]; - if (header_auth->key.data != NULL) { - if (strncasecmp(header_auth->val.data, "Splunk ", 7) == 0) { - ctx->ingested_auth_header = header_auth->val.data; - ctx->ingested_auth_header_len = header_auth->val.len; - } - } - - extra_size = session->parser.headers_extra_count; - if (extra_size > 0) { - for (i = 0; i < extra_size; i++) { - headers_extra = &session->parser.headers_extra[i]; - if (headers_extra->key.len == 16 && - strncasecmp(headers_extra->key.data, "Content-Encoding", 16) == 0) { - if (headers_extra->val.len == 4 && - strncasecmp(headers_extra->val.data, "gzip", 4) == 0) { - flb_plg_debug(ctx->ins, "body is gzipped"); - gzip_compressed = FLB_TRUE; - } - } - } - } - - if (gzip_compressed == FLB_TRUE) { - ret = flb_gzip_uncompress((void *) request->data.data, request->data.len, - &gz_data, &gz_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "gzip uncompress is failed"); - return -1; - } - - ret = handle_hec_payload(ctx, type, tag, gz_data, gz_size, - remote_addr, remote_addr_len); - flb_free(gz_data); - } - else { - ret = handle_hec_payload(ctx, type, tag, request->data.data, request->data.len, - remote_addr, remote_addr_len); - } - - return ret; -} - -static int process_hec_raw_payload(struct flb_splunk *ctx, struct splunk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request, - const char *remote_addr, - size_t remote_addr_len) -{ - int ret = -1; - struct mk_http_header *header; - struct mk_http_header *header_auth; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -2; - } - else if (header->val.len != 10 || - strncasecmp(header->val.data, "text/plain", 10) != 0) { - /* Not necessary to specify content-type for Splunk HEC. */ - flb_plg_debug(ctx->ins, "Mark as unknown type for ingested payloads"); - } - - if (request->data.len <= 0 && !mk_http_parser_is_content_chunked(&session->parser)) { - send_json_message_response(conn, 400, "{\"text\":\"No data\",\"code\":5}"); - return -2; - } - - header_auth = &session->parser.headers[MK_HEADER_AUTHORIZATION]; - if (header_auth->key.data != NULL) { - if (strncasecmp(header_auth->val.data, "Splunk ", 7) == 0) { - ctx->ingested_auth_header = header_auth->val.data; - ctx->ingested_auth_header_len = header_auth->val.len; - } - } - - /* Always handle as raw type of payloads here */ - ret = process_raw_payload_pack(ctx, tag, request->data.data, request->data.len, - remote_addr, remote_addr_len); - - return ret; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret; - int len; - char *uri; - char *qs; - char *original_data = NULL; - size_t original_data_size = 0; - char *out_chunked = NULL; - size_t out_chunked_size = 0; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - char *hval = NULL; - size_t hlen = 0; - const char *peer; - const char *remote_addr = NULL; - size_t remote_addr_len = 0; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Refer the tag at first*/ - if (ctx->ins->tag && !ctx->ins->tag_default) { - tag = flb_sds_create(ctx->ins->tag); - if (tag == NULL) { - mk_mem_free(uri); - return -1; - } - } - else { - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - /* New tag skipping the URI '/' */ - tag = flb_sds_create_len(&uri[1], len - 1); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (request->host.data == NULL && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method == MK_METHOD_GET) { - /* Handle health monitoring of splunk hec endpoint for load balancers */ - if (strcasecmp(uri, "/services/collector/health") == 0) { - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":200}"); - } - else { - send_response(conn, 400, "error: invalid HTTP endpoint\n"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - /* Under services/collector endpoints are required for - * authentication if provided splunk_token */ - ret = validate_auth_header(ctx, request); - if (ret < 0){ - send_response(conn, 401, "error: unauthorized\n"); - if (ret == SPLUNK_AUTH_MISSING_CRED) { - flb_plg_warn(ctx->ins, "missing credentials in request headers"); - } - else if (ret == SPLUNK_AUTH_UNAUTHORIZED) { - flb_plg_warn(ctx->ins, "wrong credentials in request headers"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - - /* If the request contains chunked transfer encoded data, decode it */\ - if (mk_http_parser_is_content_chunked(&session->parser)) { - ret = mk_http_parser_chunked_decode(&session->parser, - conn->buf_data, - conn->buf_len, - &out_chunked, - &out_chunked_size); - if (ret == -1) { - flb_plg_error(ctx->ins, "failed to decode chunked data"); - send_response(conn, 400, "error: invalid chunked data\n"); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return -1; - } - - /* Update the request data */ - original_data = request->data.data; - original_data_size = request->data.len; - - /* assign the chunked one */ - request->data.data = out_chunked; - request->data.len = out_chunked_size; - } - - if (http_header_lookup(HTTP_PROTOCOL_VERSION_11, request, - SPLUNK_XFF_HEADER, &hval, &hlen) == 0) { - extract_remote_address(hval, hlen, conn->connection, - (char **) &remote_addr, - &remote_addr_len); - } - if (remote_addr == NULL || remote_addr_len == 0) { - peer = flb_connection_get_remote_address(conn->connection); - if (peer != NULL) { - remote_addr = peer; - remote_addr_len = strlen(peer); - } - } - - /* Handle every ingested payload cleanly */ - flb_log_event_encoder_reset(&ctx->log_encoder); - - if (request->method == MK_METHOD_POST) { - if (strcasecmp(uri, "/services/collector/raw/1.0") == 0 || - strcasecmp(uri, "/services/collector/raw") == 0) { - ret = process_hec_raw_payload(ctx, conn, tag, session, request, - remote_addr, remote_addr_len); - - if (ret == -2) { - /* Response already sent, skip further response */ - flb_sds_destroy(tag); - mk_mem_free(uri); - if (out_chunked) { - mk_mem_free(out_chunked); - } - request->data.data = original_data; - request->data.len = original_data_size; - return -1; - } - - if (ret < 0) { - send_json_message_response(conn, 400, "{\"text\":\"Invalid data format\",\"code\":6}"); - } - else { - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":0}"); - } - } - else if (strcasecmp(uri, "/services/collector/event/1.0") == 0 || - strcasecmp(uri, "/services/collector/event") == 0 || - strcasecmp(uri, "/services/collector") == 0) { - - ret = process_hec_payload(ctx, conn, tag, session, request, - remote_addr, remote_addr_len); - if (ret == -2) { - flb_sds_destroy(tag); - mk_mem_free(uri); - - if (out_chunked) { - mk_mem_free(out_chunked); - } - request->data.data = original_data; - request->data.len = original_data_size; - - return -1; - } - - if (ret < 0) { - send_json_message_response(conn, 400, "{\"text\":\"Invalid data format\",\"code\":6}"); - } - else { - send_json_message_response(conn, 200, "{\"text\":\"Success\",\"code\":0}"); - } - } - else { - send_response(conn, 400, "error: invalid HTTP endpoint\n"); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - if (out_chunked) { - mk_mem_free(out_chunked); - } - request->data.data = original_data; - request->data.len = original_data_size; - - return -1; - } - } - else { - /* HEAD, PUT, PATCH, and DELETE methods are prohibited to use.*/ - - flb_sds_destroy(tag); - mk_mem_free(uri); - - if (out_chunked) { - mk_mem_free(out_chunked); - } - request->data.data = original_data; - request->data.len = original_data_size; - - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - if (out_chunked) { - mk_mem_free(out_chunked); - } - request->data.data = original_data; - request->data.len = original_data_size; - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int splunk_prot_handle_error(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} - - - - - - - /* New gen HTTP server */ static int send_response_ng(struct flb_http_response *response, @@ -1173,6 +488,9 @@ static int send_response_ng(struct flb_http_response *response, else if (http_status == 400) { flb_http_response_set_message(response, "Bad Request"); } + else if (http_status == 401) { + flb_http_response_set_message(response, "Unauthorized"); + } if (message != NULL) { flb_http_response_set_body(response, @@ -1203,6 +521,9 @@ static int send_json_message_response_ng(struct flb_http_response *response, else if (http_status == 400) { flb_http_response_set_message(response, "Bad Request"); } + else if (http_status == 401) { + flb_http_response_set_message(response, "Unauthorized"); + } flb_http_response_set_header(response, "content-type", 0, @@ -1405,10 +726,12 @@ int splunk_prot_handle_ng(struct flb_http_request *request, parent_session = (struct flb_http_server_session *) request->stream->parent; if (parent_session != NULL) { - if (http_header_lookup(HTTP_PROTOCOL_VERSION_20, request, - SPLUNK_XFF_HEADER, &hval, &hlen) == 0) { + hval = flb_http_request_get_header(request, SPLUNK_XFF_HEADER); + + if (hval != NULL) { + hlen = strlen(hval); extract_remote_address(hval, hlen, parent_session->connection, - (char **) &remote_addr, + &remote_addr, &remote_addr_len); } if (remote_addr == NULL || remote_addr_len == 0) { diff --git a/plugins/in_splunk/splunk_prot.h b/plugins/in_splunk/splunk_prot.h index 5c965155686..39b439bd89a 100644 --- a/plugins/in_splunk/splunk_prot.h +++ b/plugins/in_splunk/splunk_prot.h @@ -29,14 +29,6 @@ #include -int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int splunk_prot_handle_error(struct flb_splunk *ctx, struct splunk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - int splunk_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response); From 993b7588f962bc9b6b1dbbde350ab60a1e0208c0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:24:49 -0600 Subject: [PATCH 07/42] in_elasticsearch: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/in_elasticsearch/CMakeLists.txt | 1 - plugins/in_elasticsearch/in_elasticsearch.c | 166 +---- plugins/in_elasticsearch/in_elasticsearch.h | 14 - .../in_elasticsearch_bulk_conn.c | 329 ---------- .../in_elasticsearch_bulk_conn.h | 55 -- .../in_elasticsearch_bulk_prot.c | 571 +----------------- .../in_elasticsearch_bulk_prot.h | 13 - .../in_elasticsearch_config.c | 33 +- 8 files changed, 37 insertions(+), 1145 deletions(-) delete mode 100644 plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c delete mode 100644 plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h diff --git a/plugins/in_elasticsearch/CMakeLists.txt b/plugins/in_elasticsearch/CMakeLists.txt index 50a472f6afb..5d3750e8775 100644 --- a/plugins/in_elasticsearch/CMakeLists.txt +++ b/plugins/in_elasticsearch/CMakeLists.txt @@ -5,7 +5,6 @@ endif() set(src in_elasticsearch.c in_elasticsearch_config.c - in_elasticsearch_bulk_conn.c in_elasticsearch_bulk_prot.c ) diff --git a/plugins/in_elasticsearch/in_elasticsearch.c b/plugins/in_elasticsearch/in_elasticsearch.c index 55955d45c40..4ea76e76617 100644 --- a/plugins/in_elasticsearch/in_elasticsearch.c +++ b/plugins/in_elasticsearch/in_elasticsearch.c @@ -19,50 +19,12 @@ #include -#include #include #include #include "in_elasticsearch.h" #include "in_elasticsearch_config.h" #include "in_elasticsearch_bulk_prot.h" -#include "in_elasticsearch_bulk_conn.h" - -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_elasticsearch_bulk_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct in_elasticsearch_bulk_conn *conn; - struct flb_in_elasticsearch *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", - connection->fd); - - conn = in_elasticsearch_bulk_conn_add(connection, ctx); - - if (conn == NULL) { - flb_downstream_conn_release(connection); - - return -1; - } - - return 0; -} static void bytes_to_groupname(unsigned char *data, char *buf, size_t len) { int index; @@ -92,11 +54,12 @@ static void bytes_to_nodename(unsigned char *data, char *buf, size_t len) { static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; - int ret; - struct flb_in_elasticsearch *ctx; + int ret; + struct flb_in_elasticsearch *ctx; unsigned char rand[16]; + struct flb_http_server_options http_server_options; + (void) config; (void) data; /* Create context and basic conf */ @@ -105,8 +68,6 @@ static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, return -1; } - ctx->collector_id = -1; - /* Populate context with config map defaults and incoming properties */ ret = flb_input_config_map_set(ins, (void *) ctx); if (ret == -1) { @@ -118,8 +79,6 @@ static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, /* Set the context */ flb_input_set_context(ins, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - if (flb_random_bytes(rand, 16)) { flb_plg_error(ctx->ins, "cannot generate cluster name"); in_elasticsearch_config_destroy(ctx); @@ -136,84 +95,41 @@ static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, bytes_to_nodename(rand, ctx->node_name, 12); - if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_VERSION_AUTODETECT, - (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), - NULL, - ins->host.listen, - ins->host.port, - ins->tls, - ins->flags, - &ins->net_setup, - flb_input_event_loop_get(ins), - ins->config, - (void *) ctx); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not initialize http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - in_elasticsearch_config_destroy(ctx); - - return -1; + ret = flb_input_http_server_options_init( + &http_server_options, + ins, + (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), + in_elasticsearch_bulk_prot_handle_ng, + ctx); + if (ret == 0) { + ret = flb_http_server_init_with_options(&ctx->http_server, + &http_server_options); + + if (ret == 0) { + ret = flb_http_server_start(&ctx->http_server); } - ret = flb_http_server_start(&ctx->http_server); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not start http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - in_elasticsearch_config_destroy(ctx); - - return -1; + if (ret == 0) { + ret = flb_input_downstream_set(ctx->http_server.downstream, ins); } - - flb_http_server_set_buffer_max_size(&ctx->http_server, ctx->buffer_max_size); - - ctx->http_server.request_callback = in_elasticsearch_bulk_prot_handle_ng; - - flb_input_downstream_set(ctx->http_server.downstream, ctx->ins); } - else { - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - in_elasticsearch_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_elasticsearch_bulk_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_ELASTICSEARCH input plugin"); - in_elasticsearch_config_destroy(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not initialize http server on %s:%u. Aborting", + ins->host.listen, ins->host.port); - return -1; - } + in_elasticsearch_config_destroy(ctx); - ctx->collector_id = ret; + return -1; } + flb_plg_info(ctx->ins, "listening on %s:%u with %i worker%s", + ins->host.listen, + ins->host.port, + ctx->http_server.workers, + ctx->http_server.workers == 1 ? "" : "s"); + return 0; } @@ -234,24 +150,6 @@ static int in_elasticsearch_bulk_exit(void *data, struct flb_config *config) /* Configuration properties map */ static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "http2", "true", - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, enable_http2), - "Enable HTTP/2 support" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, buffer_max_size), - "Set the maximum size of buffer" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, buffer_chunk_size), - "Set the buffer chunk size" - }, - { FLB_CONFIG_MAP_STR, "tag_key", NULL, 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, tag_key), @@ -286,11 +184,11 @@ struct flb_input_plugin in_elasticsearch_plugin = { .description = "HTTP Endpoints for Elasticsearch (Bulk API)", .cb_init = in_elasticsearch_bulk_init, .cb_pre_run = NULL, - .cb_collect = in_elasticsearch_bulk_collect, + .cb_collect = NULL, .cb_flush_buf = NULL, .cb_pause = NULL, .cb_resume = NULL, .cb_exit = in_elasticsearch_bulk_exit, .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS + .flags = FLB_INPUT_NET_SERVER | FLB_INPUT_HTTP_SERVER | FLB_IO_OPT_TLS }; diff --git a/plugins/in_elasticsearch/in_elasticsearch.h b/plugins/in_elasticsearch/in_elasticsearch.h index 947327298d4..ab4de3ad852 100644 --- a/plugins/in_elasticsearch/in_elasticsearch.h +++ b/plugins/in_elasticsearch/in_elasticsearch.h @@ -20,7 +20,6 @@ #ifndef FLB_IN_ELASTICSEARCH_H #define FLB_IN_ELASTICSEARCH_H -#include #include #include #include @@ -48,20 +47,7 @@ struct flb_in_elasticsearch { struct flb_input_instance *ins; - /* New gen HTTP server */ - int enable_http2; struct flb_http_server http_server; - - /* Legacy HTTP server */ - int collector_id; - - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; }; diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c b/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c deleted file mode 100644 index 02df06cc706..00000000000 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.c +++ /dev/null @@ -1,329 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include "in_elasticsearch.h" -#include "in_elasticsearch_bulk_conn.h" -#include "in_elasticsearch_bulk_prot.h" - -static void in_elasticsearch_bulk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int elasticsearch_conn_buffer_realloc(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, size_t size) -{ - char *tmp; - - /* Perform realloc */ - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - flb_plg_error(ctx->ins, "could not perform realloc for size %zu", size); - return -1; - } - - /* Update buffer info */ - conn->buf_data = tmp; - conn->buf_size = size; - - /* Keep NULL termination */ - conn->buf_data[conn->buf_len] = '\0'; - - /* Reset parser state */ - mk_http_parser_init(&conn->session.parser); - - return 0; -} - -static int in_elasticsearch_bulk_conn_event(void *data) -{ - int ret; - int status; - size_t size; - ssize_t available; - ssize_t bytes; - size_t request_len; - struct flb_connection *connection; - struct in_elasticsearch_bulk_conn *conn; - struct mk_event *event; - struct flb_in_elasticsearch *ctx; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - ret = elasticsearch_conn_buffer_realloc(ctx, conn, size); - if (ret == -1) { - flb_errno(); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - in_elasticsearch_bulk_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* - * Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - /* Use the last parser position as the request length */ - request_len = mk_http_parser_request_size(&conn->session.parser, - conn->buf_data, - conn->buf_len); - - if (request_len == -1 || (request_len > conn->buf_len)) { - /* Unexpected but let's make sure things are safe */ - conn->buf_len = 0; - flb_plg_debug(ctx->ins, "request length exceeds buffer length, closing connection"); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - /* If we have extra bytes in our bytes, adjust the extra bytes */ - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - conn->buf_len = 0; - } - - /* - * Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - } - else if (status == MK_HTTP_PARSER_ERROR) { - in_elasticsearch_bulk_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - in_elasticsearch_bulk_conn_del(conn); - return -1; - } - - return 0; - -} - -static void in_elasticsearch_bulk_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void in_elasticsearch_bulk_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct in_elasticsearch_bulk_conn *in_elasticsearch_bulk_conn_add(struct flb_connection *connection, - struct flb_in_elasticsearch *ctx) -{ - struct in_elasticsearch_bulk_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct in_elasticsearch_bulk_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = in_elasticsearch_bulk_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - - flb_free(conn->buf_data); - flb_free(conn); - - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - in_elasticsearch_bulk_conn_session_init(&conn->session, ctx->server, conn->connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - in_elasticsearch_bulk_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - - return conn; -} - -int in_elasticsearch_bulk_conn_del(struct in_elasticsearch_bulk_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void in_elasticsearch_bulk_conn_release_all(struct flb_in_elasticsearch *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct in_elasticsearch_bulk_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct in_elasticsearch_bulk_conn, _head); - in_elasticsearch_bulk_conn_del(conn); - } -} diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h b/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h deleted file mode 100644 index 9647f2075c1..00000000000 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_conn.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ELASTICSEARCH_BULK_CONN -#define FLB_IN_ELASTICSEARCH_BULK_CONN - -#include -#include - -#include -#include -#include - -struct in_elasticsearch_bulk_conn { - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_es_bulk->connections */ -}; - -struct in_elasticsearch_bulk_conn *in_elasticsearch_bulk_conn_add(struct flb_connection *connection, - struct flb_in_elasticsearch *ctx); -int in_elasticsearch_bulk_conn_del(struct in_elasticsearch_bulk_conn *conn); -void in_elasticsearch_bulk_conn_release_all(struct flb_in_elasticsearch *ctx); - - -#endif diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c index 763cf9993e8..00949813328 100644 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c +++ b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c @@ -29,221 +29,11 @@ #include #include "in_elasticsearch.h" -#include "in_elasticsearch_bulk_conn.h" #include "in_elasticsearch_bulk_prot.h" #define HTTP_CONTENT_JSON 0 #define HTTP_CONTENT_NDJSON 1 -static int send_empty_response(struct in_elasticsearch_bulk_conn *conn, int http_status) -{ - size_t sent; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n\r\n"); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_json_message_response(struct in_elasticsearch_bulk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int send_version_message_response(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, int http_status) -{ - size_t sent; - int len; - flb_sds_t out; - flb_sds_t resp; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - resp = flb_sds_create_size(384); - if (!resp) { - flb_sds_destroy(out); - return -1; - } - - flb_sds_printf(&resp, - ES_VERSION_RESPONSE_TEMPLATE, - ctx->es_version); - - len = flb_sds_len(resp); - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, resp); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(resp); - flb_sds_destroy(out); - - return 0; -} - -static int send_dummy_sniffer_response(struct in_elasticsearch_bulk_conn *conn, int http_status, - struct flb_in_elasticsearch *ctx) -{ - size_t sent; - int len; - flb_sds_t out; - flb_sds_t resp; - flb_sds_t hostname; - - if (ctx->hostname != NULL) { - hostname = ctx->hostname; - } - else { - hostname = "localhost"; - } - - out = flb_sds_create_size(384); - if (!out) { - return -1; - } - - resp = flb_sds_create_size(384); - if (!resp) { - flb_sds_destroy(out); - return -1; - } - - flb_sds_printf(&resp, - ES_NODES_TEMPLATE, - ctx->cluster_name, ctx->node_name, - hostname, ctx->tcp_port, ctx->buffer_max_size); - - len = flb_sds_len(resp) ; - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - len, resp); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(resp); - flb_sds_destroy(out); - - return 0; -} - -static int send_response(struct in_elasticsearch_bulk_conn *conn, int http_status, char *message) -{ - size_t sent; - int len; - flb_sds_t out; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check this operations result */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - /* implements functionality to get tag from key in record */ static flb_sds_t tag_key(struct flb_in_elasticsearch *ctx, msgpack_object *map) { @@ -573,359 +363,6 @@ static ssize_t parse_payload_ndjson(struct flb_in_elasticsearch *ctx, flb_sds_t return 0; } -static int process_payload(struct flb_in_elasticsearch *ctx, struct in_elasticsearch_bulk_conn *conn, - flb_sds_t tag, - struct mk_http_session *session, - struct mk_http_request *request, - flb_sds_t bulk_statuses) -{ - int type = -1; - int i = 0; - int ret = 0; - struct mk_http_header *header; - int extra_size = -1; - struct mk_http_header *headers_extra; - int gzip_compressed = FLB_FALSE; - void *gz_data = NULL; - size_t gz_size = -1; - char *out_chunked = NULL; - size_t out_chunked_size = 0; - char *payload_buf; - size_t payload_size; - - header = &session->parser.headers[MK_HEADER_CONTENT_TYPE]; - if (header->key.data == NULL) { - send_response(conn, 400, "error: header 'Content-Type' is not set\n"); - return -1; - } - - if (header->val.len >= 20 && - strncasecmp(header->val.data, "application/x-ndjson", 20) == 0) { - type = HTTP_CONTENT_NDJSON; - } - - if (header->val.len >= 16 && - strncasecmp(header->val.data, "application/json", 16) == 0) { - type = HTTP_CONTENT_JSON; - } - - if (type == -1) { - send_response(conn, 400, "error: invalid 'Content-Type'\n"); - return -1; - } - - if (request->data.len <= 0 && !mk_http_parser_is_content_chunked(&session->parser)) { - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - extra_size = session->parser.headers_extra_count; - if (extra_size > 0) { - for (i = 0; i < extra_size; i++) { - headers_extra = &session->parser.headers_extra[i]; - if (headers_extra->key.len == 16 && - strncasecmp(headers_extra->key.data, "Content-Encoding", 16) == 0) { - if (headers_extra->val.len == 4 && - strncasecmp(headers_extra->val.data, "gzip", 4) == 0) { - flb_debug("[elasticsearch_bulk_prot] body is gzipped"); - gzip_compressed = FLB_TRUE; - } - } - } - } - - if (type == HTTP_CONTENT_NDJSON || type == HTTP_CONTENT_JSON) { - /* Check if the data is chunked */ - payload_buf = NULL; - payload_size = 0; - - if (mk_http_parser_is_content_chunked(&session->parser)) { - ret = mk_http_parser_chunked_decode(&session->parser, - conn->buf_data, - conn->buf_len, - &out_chunked, - &out_chunked_size); - - if (ret == -1) { - send_response(conn, 400, "error: invalid chunked data\n"); - return -1; - } - - payload_buf = out_chunked; - payload_size = out_chunked_size; - } - else { - payload_buf = request->data.data; - payload_size = request->data.len; - } - - if (gzip_compressed == FLB_TRUE) { - ret = flb_gzip_uncompress((void *) payload_buf, payload_size, - &gz_data, &gz_size); - if (ret == -1) { - flb_error("[elasticsearch_bulk_prot] gzip uncompress is failed"); - return -1; - } - parse_payload_ndjson(ctx, tag, gz_data, gz_size, bulk_statuses); - flb_free(gz_data); - } - else { - parse_payload_ndjson(ctx, tag, payload_buf, payload_size, bulk_statuses); - } - } - - /* release chunked data if has been set */ - if (out_chunked) { - mk_mem_free(out_chunked); - } - - return 0; -} - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - -/* - * Handle an incoming request. It perform extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int in_elasticsearch_bulk_prot_handle(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret; - int len; - char *uri; - char *qs; - off_t diff; - flb_sds_t tag; - struct mk_http_header *header; - flb_sds_t bulk_statuses = NULL; - flb_sds_t bulk_response = NULL; - char *error_str = NULL; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - /* Try to match a query string so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Refer the tag at first*/ - if (ctx->ins->tag && !ctx->ins->tag_default) { - tag = flb_sds_create(ctx->ins->tag); - if (tag == NULL) { - mk_mem_free(uri); - return -1; - } - } - else { - /* Compose the query string using the URI */ - len = strlen(uri); - - if (len == 1) { - tag = NULL; /* use default tag */ - } - else { - /* New tag skipping the URI '/' */ - tag = flb_sds_create_len(&uri[1], len - 1); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - } - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - if (request->method == MK_METHOD_HEAD) { - send_empty_response(conn, 200); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_PUT) { - send_json_message_response(conn, 200, "{}"); - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_GET) { - if (strncmp(uri, "/_nodes/http", 12) == 0) { - send_dummy_sniffer_response(conn, 200, ctx); - } - else if (strlen(uri) == 1 && strncmp(uri, "/", 1) == 0) { - send_version_message_response(ctx, conn, 200); - } - else { - send_json_message_response(conn, 200, "{}"); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - return 0; - } - - if (request->method == MK_METHOD_POST) { - if (strncmp(uri, "/_bulk", 6) == 0) { - bulk_statuses = flb_sds_create_size(ctx->buffer_max_size); - if (!bulk_statuses) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - bulk_response = flb_sds_create_size(ctx->buffer_max_size); - if (!bulk_response) { - flb_sds_destroy(bulk_statuses); - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - } - else { - flb_sds_destroy(tag); - mk_mem_free(uri); - - send_response(conn, 400, "error: invalid HTTP endpoint\n"); - - return -1; - } - } - - if (request->method != MK_METHOD_POST && - request->method != MK_METHOD_GET && - request->method != MK_METHOD_HEAD && - request->method != MK_METHOD_PUT) { - - if (bulk_statuses) { - flb_sds_destroy(bulk_statuses); - } - if (bulk_response) { - flb_sds_destroy(bulk_response); - } - - flb_sds_destroy(tag); - mk_mem_free(uri); - - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - ret = process_payload(ctx, conn, tag, session, request, bulk_statuses); - flb_sds_destroy(tag); - - len = flb_sds_len(bulk_statuses); - if (flb_sds_alloc(bulk_response) < len + 27) { - bulk_response = flb_sds_increase(bulk_response, len + 27 - flb_sds_alloc(bulk_response)); - } - error_str = strstr(bulk_statuses, "\"status\":40"); - if (error_str){ - flb_sds_cat_safe(&bulk_response, "{\"errors\":true,\"items\":[", 24); - } - else { - flb_sds_cat_safe(&bulk_response, "{\"errors\":false,\"items\":[", 25); - } - flb_sds_cat_safe(&bulk_response, bulk_statuses, flb_sds_len(bulk_statuses)); - flb_sds_cat_safe(&bulk_response, "]}", 2); - send_response(conn, 200, bulk_response); - - mk_mem_free(uri); - flb_sds_destroy(bulk_statuses); - flb_sds_destroy(bulk_response); - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int in_elasticsearch_bulk_prot_handle_error(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} - - - - /* New gen HTTP server */ static int send_response_ng(struct flb_http_response *response, int http_status, @@ -1024,7 +461,7 @@ static int send_dummy_sniffer_response_ng(struct flb_http_response *response, flb_sds_printf(&resp, ES_NODES_TEMPLATE, ctx->cluster_name, ctx->node_name, - hostname, ctx->tcp_port, ctx->buffer_max_size); + hostname, ctx->tcp_port, ctx->ins->http_server_config->buffer_max_size); send_json_response_ng(response, http_status, resp); @@ -1114,13 +551,13 @@ int in_elasticsearch_bulk_prot_handle_ng(struct flb_http_request *request, } else if (request->method == HTTP_METHOD_POST) { if (strcmp(request->path, "/_bulk") == 0) { - bulk_statuses = flb_sds_create_size(context->buffer_max_size); + bulk_statuses = flb_sds_create_size(context->ins->http_server_config->buffer_max_size); if (bulk_statuses == NULL) { return -1; } - bulk_response = flb_sds_create_size(context->buffer_max_size); + bulk_response = flb_sds_create_size(context->ins->http_server_config->buffer_max_size); if (bulk_response == NULL) { flb_sds_destroy(bulk_statuses); @@ -1175,4 +612,4 @@ int in_elasticsearch_bulk_prot_handle_ng(struct flb_http_request *request, } return 0; -} \ No newline at end of file +} diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h index ca6472669b9..4195de16d92 100644 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h +++ b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.h @@ -20,8 +20,6 @@ #ifndef FLB_IN_ELASTICSEARCH_BULK_PROT #define FLB_IN_ELASTICSEARCH_BULK_PROT -#include "in_elasticsearch_bulk_conn.h" - #define ES_VERSION_RESPONSE_TEMPLATE \ "{\"version\":{\"number\":\"%s\",\"build_flavor\":\"Fluent Bit OSS\"},\"tagline\":\"Fluent Bit's Bulk API compatible endpoint\"}" @@ -29,17 +27,6 @@ "\"nodes\":{\"%s\":{\"name\":\"%s\",\"version\":\"8.0.0\"," \ "\"http\":{\"publish_address\":\"%s:%s\",\"max_content_length_in_bytes\":%ld}}}}" -int in_elasticsearch_bulk_prot_handle(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int in_elasticsearch_bulk_prot_handle_error(struct flb_in_elasticsearch *ctx, - struct in_elasticsearch_bulk_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - - int in_elasticsearch_bulk_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response); diff --git a/plugins/in_elasticsearch/in_elasticsearch_config.c b/plugins/in_elasticsearch/in_elasticsearch_config.c index 195db093b3e..2464efa7772 100644 --- a/plugins/in_elasticsearch/in_elasticsearch_config.c +++ b/plugins/in_elasticsearch/in_elasticsearch_config.c @@ -21,8 +21,6 @@ #include "in_elasticsearch.h" #include "in_elasticsearch_config.h" -#include "in_elasticsearch_bulk_conn.h" - struct flb_in_elasticsearch *in_elasticsearch_config_create(struct flb_input_instance *ins) { int ret; @@ -35,7 +33,6 @@ struct flb_in_elasticsearch *in_elasticsearch_config_create(struct flb_input_ins return NULL; } ctx->ins = ins; - mk_list_init(&ctx->connections); /* Load the config map */ ret = flb_input_config_map_set(ins, (void *) ctx); @@ -51,14 +48,6 @@ struct flb_in_elasticsearch *in_elasticsearch_config_create(struct flb_input_ins snprintf(port, sizeof(port) - 1, "%d", ins->host.port); ctx->tcp_port = flb_sds_create(port); - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); if (ctx->log_encoder == NULL) { flb_plg_error(ctx->ins, "event encoder initialization error"); @@ -88,27 +77,7 @@ int in_elasticsearch_config_destroy(struct flb_in_elasticsearch *ctx) flb_log_event_encoder_destroy(ctx->log_encoder); - /* release all connections */ - in_elasticsearch_bulk_conn_release_all(ctx); - - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->enable_http2) { - flb_http_server_destroy(&ctx->http_server); - } - - if (ctx->server) { - flb_free(ctx->server); - } + flb_http_server_destroy(&ctx->http_server); flb_sds_destroy(ctx->listen); flb_sds_destroy(ctx->tcp_port); From 224a8995ac185f0861dd23b28382f0d43c45f952 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:25:18 -0600 Subject: [PATCH 08/42] in_opentelemetry: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/CMakeLists.txt | 1 - plugins/in_opentelemetry/http_conn.c | 323 ------------- plugins/in_opentelemetry/http_conn.h | 57 --- plugins/in_opentelemetry/opentelemetry.c | 158 ++----- plugins/in_opentelemetry/opentelemetry.h | 13 - .../in_opentelemetry/opentelemetry_config.c | 33 +- plugins/in_opentelemetry/opentelemetry_prot.c | 434 ------------------ plugins/in_opentelemetry/opentelemetry_prot.h | 9 - 8 files changed, 30 insertions(+), 998 deletions(-) delete mode 100644 plugins/in_opentelemetry/http_conn.c delete mode 100644 plugins/in_opentelemetry/http_conn.h diff --git a/plugins/in_opentelemetry/CMakeLists.txt b/plugins/in_opentelemetry/CMakeLists.txt index 7eb159c15a0..5b31b406906 100644 --- a/plugins/in_opentelemetry/CMakeLists.txt +++ b/plugins/in_opentelemetry/CMakeLists.txt @@ -3,7 +3,6 @@ if(NOT FLB_METRICS) endif() set(src - http_conn.c opentelemetry.c opentelemetry_prot.c opentelemetry_logs.c diff --git a/plugins/in_opentelemetry/http_conn.c b/plugins/in_opentelemetry/http_conn.c deleted file mode 100644 index 09d89f85f48..00000000000 --- a/plugins/in_opentelemetry/http_conn.c +++ /dev/null @@ -1,323 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include "opentelemetry.h" -#include "http_conn.h" -#include "opentelemetry_prot.h" - -static void opentelemetry_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request); - -static int opentelemetry_conn_buffer_realloc(struct flb_opentelemetry *ctx, - struct http_conn *conn, size_t size) -{ - char *tmp; - - /* Perform realloc */ - tmp = flb_realloc(conn->buf_data, size); - if (!tmp) { - flb_errno(); - flb_plg_error(ctx->ins, "could not perform realloc for size %zu", size); - return -1; - } - - /* Update buffer info */ - conn->buf_data = tmp; - conn->buf_size = size; - - /* Keep NULL termination */ - conn->buf_data[conn->buf_len] = '\0'; - - /* Reset parser state */ - mk_http_parser_init(&conn->session.parser); - - return 0; -} - -static int opentelemetry_conn_event(void *data) -{ - int ret; - int status; - size_t size; - ssize_t available; - ssize_t bytes; - size_t request_len; - struct http_conn *conn; - struct mk_event *event; - struct flb_opentelemetry *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; - - conn = connection->user_data; - - ctx = conn->ctx; - - event = &connection->event; - - if (event->mask & MK_EVENT_READ) { - available = (conn->buf_size - conn->buf_len) - 1; - if (available < 1) { - if (conn->buf_size + ctx->buffer_chunk_size > ctx->buffer_max_size) { - flb_plg_trace(ctx->ins, - "fd=%i incoming data exceed limit (%zu KB)", - event->fd, (ctx->buffer_max_size / 1024)); - opentelemetry_conn_del(conn); - return -1; - } - - size = conn->buf_size + ctx->buffer_chunk_size; - ret = opentelemetry_conn_buffer_realloc(ctx, conn, size); - if (ret == -1) { - flb_errno(); - opentelemetry_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "fd=%i buffer realloc %i -> %zu", - event->fd, conn->buf_size, size); - - available = (conn->buf_size - conn->buf_len) - 1; - } - - /* Read data */ - bytes = flb_io_net_read(connection, - (void *) &conn->buf_data[conn->buf_len], - available); - - if (bytes <= 0) { - flb_plg_trace(ctx->ins, "fd=%i closed connection", event->fd); - opentelemetry_conn_del(conn); - return -1; - } - - flb_plg_trace(ctx->ins, "read()=%zi pre_len=%i now_len=%zi", - bytes, conn->buf_len, conn->buf_len + bytes); - conn->buf_len += bytes; - conn->buf_data[conn->buf_len] = '\0'; - - status = mk_http_parser(&conn->request, &conn->session.parser, - conn->buf_data, conn->buf_len, conn->session.server); - - if (status == MK_HTTP_PARSER_OK) { - /* Do more logic parsing and checks for this request */ - opentelemetry_prot_handle(ctx, conn, &conn->session, &conn->request); - - /* - * Evict the processed request from the connection buffer and reinitialize - * the HTTP parser. - */ - - /* Use the last parser position as the request length */ - request_len = mk_http_parser_request_size(&conn->session.parser, - conn->buf_data, - conn->buf_len); - - if (request_len == -1 || (request_len > conn->buf_len)) { - /* Unexpected but let's make sure things are safe */ - conn->buf_len = 0; - flb_plg_debug(ctx->ins, "request length exceeds buffer length, closing connection"); - opentelemetry_conn_del(conn); - return -1; - } - - /* If we have extra bytes in our bytes, adjust the extra bytes */ - if (0 < (conn->buf_len - request_len)) { - memmove(conn->buf_data, &conn->buf_data[request_len], - conn->buf_len - request_len); - - conn->buf_data[conn->buf_len - request_len] = '\0'; - conn->buf_len -= request_len; - } - else { - memset(conn->buf_data, 0, request_len); - conn->buf_len = 0; - } - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - opentelemetry_conn_request_init(&conn->session, &conn->request); - } - else if (status == MK_HTTP_PARSER_ERROR) { - opentelemetry_prot_handle_error(ctx, conn, &conn->session, &conn->request); - - /* Reinitialize the parser so the next request is properly - * handled, the additional memset intends to wipe any left over data - * from the headers parsed in the previous request. - */ - memset(&conn->session.parser, 0, sizeof(struct mk_http_parser)); - mk_http_parser_init(&conn->session.parser); - opentelemetry_conn_request_init(&conn->session, &conn->request); - } - - /* FIXME: add Protocol handler here */ - return bytes; - } - - if (event->mask & MK_EVENT_CLOSE) { - flb_plg_trace(ctx->ins, "fd=%i hangup", event->fd); - opentelemetry_conn_del(conn); - return -1; - } - - return 0; - -} - -static void opentelemetry_conn_session_init(struct mk_http_session *session, - struct mk_server *server, - int client_fd) -{ - /* Alloc memory for node */ - session->_sched_init = MK_TRUE; - session->pipelined = MK_FALSE; - session->counter_connections = 0; - session->close_now = MK_FALSE; - session->status = MK_REQUEST_STATUS_INCOMPLETE; - session->server = server; - session->socket = client_fd; - - /* creation time in unix time */ - session->init_time = time(NULL); - - session->channel = mk_channel_new(MK_CHANNEL_SOCKET, session->socket); - session->channel->io = session->server->network; - - /* Init session request list */ - mk_list_init(&session->request_list); - - /* Initialize the parser */ - mk_http_parser_init(&session->parser); -} - -static void opentelemetry_conn_request_init(struct mk_http_session *session, - struct mk_http_request *request) -{ - memset(request, 0, sizeof(struct mk_http_request)); - - mk_http_request_init(session, request, session->server); - - request->in_headers.type = MK_STREAM_IOV; - request->in_headers.dynamic = MK_FALSE; - request->in_headers.cb_consumed = NULL; - request->in_headers.cb_finished = NULL; - request->in_headers.stream = &request->stream; - - mk_list_add(&request->in_headers._head, &request->stream.inputs); - - request->session = session; -} - -struct http_conn *opentelemetry_conn_add(struct flb_connection *connection, - struct flb_opentelemetry *ctx) -{ - struct http_conn *conn; - int ret; - - conn = flb_calloc(1, sizeof(struct http_conn)); - if (!conn) { - flb_errno(); - return NULL; - } - conn->connection = connection; - - /* Set data for the event-loop */ - MK_EVENT_NEW(&connection->event); - - connection->user_data = conn; - connection->event.type = FLB_ENGINE_EV_CUSTOM; - connection->event.handler = opentelemetry_conn_event; - - /* Connection info */ - conn->ctx = ctx; - conn->buf_len = 0; - - conn->buf_data = flb_malloc(ctx->buffer_chunk_size); - if (!conn->buf_data) { - flb_errno(); - flb_plg_error(ctx->ins, "could not allocate new connection"); - flb_free(conn); - return NULL; - } - conn->buf_size = ctx->buffer_chunk_size; - - /* Register instance into the event loop */ - ret = mk_event_add(flb_engine_evl_get(), - connection->fd, - FLB_ENGINE_EV_CUSTOM, - MK_EVENT_READ, - &connection->event); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not register new connection"); - flb_free(conn->buf_data); - flb_free(conn); - return NULL; - } - - /* Initialize HTTP Session: this is a custom context for Monkey HTTP */ - opentelemetry_conn_session_init(&conn->session, ctx->server, connection->fd); - - /* Initialize HTTP Request: this is the initial request and it will be reinitialized - * automatically after the request is handled so it can be used for the next one. - */ - opentelemetry_conn_request_init(&conn->session, &conn->request); - - /* Link connection node to parent context list */ - mk_list_add(&conn->_head, &ctx->connections); - return conn; -} - -int opentelemetry_conn_del(struct http_conn *conn) -{ - if (conn->session.channel != NULL) { - mk_channel_release(conn->session.channel); - } - - /* The downstream unregisters the file descriptor from the event-loop - * so there's nothing to be done by the plugin - */ - flb_downstream_conn_release(conn->connection); - - mk_list_del(&conn->_head); - - flb_free(conn->buf_data); - flb_free(conn); - - return 0; -} - -void opentelemetry_conn_release_all(struct flb_opentelemetry *ctx) -{ - struct mk_list *tmp; - struct mk_list *head; - struct http_conn *conn; - - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct http_conn, _head); - opentelemetry_conn_del(conn); - } -} diff --git a/plugins/in_opentelemetry/http_conn.h b/plugins/in_opentelemetry/http_conn.h deleted file mode 100644 index 88d8b94176f..00000000000 --- a/plugins/in_opentelemetry/http_conn.h +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_HTTP_CONN -#define FLB_IN_HTTP_CONN - -#include -#include -#include -#include - -#include "opentelemetry.h" - -struct http_conn { - struct mk_event event; /* Built-in event data for mk_events */ - - /* Buffer */ - char *buf_data; /* Buffer data */ - int buf_len; /* Data length */ - int buf_size; /* Buffer size */ - - /* - * Parser context: we only held one parser per connection - * which is re-used everytime we have a new request. - */ - struct mk_http_parser parser; - struct mk_http_request request; - struct mk_http_session session; - struct flb_connection *connection; - - void *ctx; /* Plugin parent context */ - struct mk_list _head; /* link to flb_opentelemetry->connections */ -}; - -struct http_conn *opentelemetry_conn_add(struct flb_connection *connection, - struct flb_opentelemetry *ctx); -int opentelemetry_conn_del(struct http_conn *conn); -void opentelemetry_conn_release_all(struct flb_opentelemetry *ctx); - - -#endif diff --git a/plugins/in_opentelemetry/opentelemetry.c b/plugins/in_opentelemetry/opentelemetry.c index f2c0430ea0a..c793a7c1333 100644 --- a/plugins/in_opentelemetry/opentelemetry.c +++ b/plugins/in_opentelemetry/opentelemetry.c @@ -19,55 +19,20 @@ #include -#include -#include #include -#include "http_conn.h" #include "opentelemetry.h" #include "opentelemetry_prot.h" #include "opentelemetry_config.h" -/* - * For a server event, the collection event means a new client have arrived, we - * accept the connection and create a new TCP instance which will wait for - * JSON map messages. - */ -static int in_opentelemetry_collect(struct flb_input_instance *ins, - struct flb_config *config, void *in_context) -{ - struct flb_connection *connection; - struct http_conn *conn; - struct flb_opentelemetry *ctx; - - ctx = in_context; - - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could not accept new connection"); - - return -1; - } - - flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); - - conn = opentelemetry_conn_add(connection, ctx); - - if (conn == NULL) { - return -1; - } - - return 0; -} - static int in_opentelemetry_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_opentelemetry *ctx; + struct flb_http_server_options http_server_options; + (void) config; (void) data; /* Create context and basic conf */ @@ -75,7 +40,6 @@ static int in_opentelemetry_init(struct flb_input_instance *ins, if (!ctx) { return -1; } - ctx->collector_id = -1; /* Populate context with config map defaults and incoming properties */ ret = flb_input_config_map_set(ins, (void *) ctx); @@ -88,86 +52,40 @@ static int in_opentelemetry_init(struct flb_input_instance *ins, /* Set the context */ flb_input_set_context(ins, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_VERSION_AUTODETECT, - (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), - NULL, - ins->host.listen, - ins->host.port, - ins->tls, - ins->flags, - &ins->net_setup, - flb_input_event_loop_get(ins), - ins->config, - (void *) ctx); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not initialize http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - opentelemetry_config_destroy(ctx); - - return -1; + ret = flb_input_http_server_options_init( + &http_server_options, + ins, + (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), + opentelemetry_prot_handle_ng, + ctx); + if (ret == 0) { + ret = flb_http_server_init_with_options(&ctx->http_server, + &http_server_options); + + if (ret == 0) { + ret = flb_http_server_start(&ctx->http_server); } - ret = flb_http_server_start(&ctx->http_server); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "could not start http server on %s:%u. Aborting", - ins->host.listen, ins->host.port); - - opentelemetry_config_destroy(ctx); - - return -1; + if (ret == 0) { + ret = flb_input_downstream_set(ctx->http_server.downstream, ins); } - - flb_http_server_set_buffer_max_size(&ctx->http_server, ctx->buffer_max_size); - - ctx->http_server.request_callback = opentelemetry_prot_handle_ng; - - flb_input_downstream_set(ctx->http_server.downstream, ctx->ins); } - else { - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ins->flags, - ctx->listen, - port, - ins->tls, - config, - &ins->net_setup); - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not initialize http server on %s:%u. Aborting", + ins->host.listen, ins->host.port); - opentelemetry_config_destroy(ctx); - - return -1; - } - - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_opentelemetry_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - opentelemetry_config_destroy(ctx); - return -1; - } + opentelemetry_config_destroy(ctx); - ctx->collector_id = ret; + return -1; } - flb_plg_info(ctx->ins, "listening on %s:%s", ctx->listen, ctx->tcp_port); + flb_plg_info(ctx->ins, "listening on %s:%u with %i worker%s", + ins->host.listen, + ins->host.port, + ctx->http_server.workers, + ctx->http_server.workers == 1 ? "" : "s"); if (ctx->successful_response_code != 200 && ctx->successful_response_code != 201 && @@ -197,12 +115,6 @@ static int in_opentelemetry_exit(void *data, struct flb_config *config) /* Configuration properties map */ static struct flb_config_map config_map[] = { - { - FLB_CONFIG_MAP_BOOL, "http2", "true", - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, enable_http2), - "Enable HTTP/2 protocol support for the OpenTelemetry receiver" - }, - { FLB_CONFIG_MAP_BOOL, "profiles_support", "false", 0, FLB_TRUE, offsetof(struct flb_opentelemetry, profile_support_enabled), @@ -216,18 +128,6 @@ static struct flb_config_map config_map[] = { "Encode profiles received as text and ingest them in the logging pipeline" }, - { - FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_max_size), - "Maximum size of the HTTP request buffer" - }, - - { - FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, - 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_chunk_size), - "Size of each buffer chunk allocated for HTTP requests" - }, - { FLB_CONFIG_MAP_STR, "tag_key", NULL, 0, FLB_TRUE, offsetof(struct flb_opentelemetry, tag_key), @@ -271,11 +171,11 @@ struct flb_input_plugin in_opentelemetry_plugin = { .description = "OpenTelemetry", .cb_init = in_opentelemetry_init, .cb_pre_run = NULL, - .cb_collect = in_opentelemetry_collect, + .cb_collect = NULL, .cb_flush_buf = NULL, .cb_pause = NULL, .cb_resume = NULL, .cb_exit = in_opentelemetry_exit, .config_map = config_map, - .flags = FLB_INPUT_NET_SERVER | FLB_IO_OPT_TLS + .flags = FLB_INPUT_NET_SERVER | FLB_INPUT_HTTP_SERVER | FLB_IO_OPT_TLS }; diff --git a/plugins/in_opentelemetry/opentelemetry.h b/plugins/in_opentelemetry/opentelemetry.h index ae740656af7..6c39d476c45 100644 --- a/plugins/in_opentelemetry/opentelemetry.h +++ b/plugins/in_opentelemetry/opentelemetry.h @@ -24,7 +24,6 @@ #include #include -#include #include #define HTTP_BUFFER_MAX_SIZE "4M" @@ -44,19 +43,7 @@ struct flb_opentelemetry { struct flb_input_instance *ins; - /* New gen HTTP server */ - int enable_http2; struct flb_http_server http_server; - - /* Legacy HTTP server */ - size_t buffer_max_size; /* Maximum buffer size */ - size_t buffer_chunk_size; /* Chunk allocation size */ - - int collector_id; /* Listener collector id */ - struct flb_downstream *downstream; /* Client manager */ - struct mk_list connections; /* linked list of connections */ - - struct mk_server *server; }; diff --git a/plugins/in_opentelemetry/opentelemetry_config.c b/plugins/in_opentelemetry/opentelemetry_config.c index 3289f065f28..c4767f6c9f1 100644 --- a/plugins/in_opentelemetry/opentelemetry_config.c +++ b/plugins/in_opentelemetry/opentelemetry_config.c @@ -18,10 +18,7 @@ */ #include -#include - #include "opentelemetry.h" -#include "http_conn.h" /* default HTTP port for OTLP/HTTP is 4318 */ #define OTLP_HTTP_PORT 4318 @@ -38,7 +35,6 @@ struct flb_opentelemetry *opentelemetry_config_create(struct flb_input_instance return NULL; } ctx->ins = ins; - mk_list_init(&ctx->connections); /* Load the config map */ ret = flb_input_config_map_set(ins, (void *) ctx); @@ -54,39 +50,12 @@ struct flb_opentelemetry *opentelemetry_config_create(struct flb_input_instance snprintf(port, sizeof(port) - 1, "%d", ins->host.port); ctx->tcp_port = flb_strdup(port); - /* HTTP Server specifics */ - ctx->server = flb_calloc(1, sizeof(struct mk_server)); - ctx->server->keep_alive = MK_TRUE; - - /* monkey detects server->workers == 0 as the server not being initialized at the - * moment so we want to make sure that it stays that way! - */ - return ctx; } int opentelemetry_config_destroy(struct flb_opentelemetry *ctx) { - /* release all connections */ - opentelemetry_conn_release_all(ctx); - - if (ctx->collector_id != -1) { - flb_input_collector_delete(ctx->collector_id, ctx->ins); - - ctx->collector_id = -1; - } - - if (ctx->downstream != NULL) { - flb_downstream_destroy(ctx->downstream); - } - - if (ctx->enable_http2) { - flb_http_server_destroy(&ctx->http_server); - } - - if (ctx->server) { - flb_free(ctx->server); - } + flb_http_server_destroy(&ctx->http_server); flb_free(ctx->listen); flb_free(ctx->tcp_port); diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index 33c3e2efe8b..4301bd9302c 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -43,8 +43,6 @@ #include "opentelemetry_logs.h" #include "opentelemetry_traces.h" -#include "http_conn.h" - #define HTTP_CONTENT_JSON 0 static int is_profiles_export_path(const char *path) @@ -61,138 +59,6 @@ static int is_profiles_export_path(const char *path) return FLB_FALSE; } -static int send_response(struct http_conn *conn, int http_status, char *message) -{ - int len; - flb_sds_t out; - size_t sent; - - out = flb_sds_create_size(256); - if (!out) { - return -1; - } - - if (message) { - len = strlen(message); - } - else { - len = 0; - } - - if (http_status == 201) { - flb_sds_printf(&out, - "HTTP/1.1 201 Created \r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 200) { - flb_sds_printf(&out, - "HTTP/1.1 200 OK\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: 0\r\n\r\n", - FLB_VERSION_STR); - } - else if (http_status == 204) { - flb_sds_printf(&out, - "HTTP/1.1 204 No Content\r\n" - "Server: Fluent Bit v%s\r\n" - "\r\n", - FLB_VERSION_STR); - } - else if (http_status == 400) { - flb_sds_printf(&out, - "HTTP/1.1 400 Bad Request\r\n" - "Server: Fluent Bit v%s\r\n" - "Content-Length: %i\r\n\r\n%s", - FLB_VERSION_STR, - len, message); - } - - /* We should check the outcome of this operation */ - flb_io_net_write(conn->connection, - (void *) out, - flb_sds_len(out), - &sent); - - flb_sds_destroy(out); - - return 0; -} - -static int process_payload_metrics(struct flb_opentelemetry *ctx, struct http_conn *conn, - flb_sds_t tag, - size_t tag_len, - flb_sds_t content_type, - struct mk_http_session *session, - struct mk_http_request *request) -{ - struct cfl_list decoded_contexts; - struct cfl_list *iterator; - struct cmt *context; - size_t offset; - int result; - - (void) conn; - (void) session; - - offset = 0; - - if (content_type != NULL && - opentelemetry_is_json_content_type(content_type) == FLB_TRUE) { - result = flb_opentelemetry_metrics_json_to_cmt(&decoded_contexts, - request->data.data, - request->data.len); - } - else { - result = cmt_decode_opentelemetry_create(&decoded_contexts, - request->data.data, - request->data.len, - &offset); - } - - if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { - flb_plg_error(ctx->ins, "could not decode metrics payload"); - return -1; - } - - if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { - cfl_list_foreach(iterator, &decoded_contexts) { - context = cfl_list_entry(iterator, struct cmt, _head); - - result = flb_input_metrics_append(ctx->ins, tag, tag_len, context); - - if (result != 0) { - flb_plg_debug(ctx->ins, "could not ingest metrics context : %d", result); - } - } - - cmt_decode_opentelemetry_destroy(&decoded_contexts); - } - - return 0; -} - - -static inline int mk_http_point_header(mk_ptr_t *h, - struct mk_http_parser *parser, int key) -{ - struct mk_http_header *header; - - header = &parser->headers[key]; - if (header->type == key) { - h->data = header->val.data; - h->len = header->val.len; - return 0; - } - else { - h->data = NULL; - h->len = -1; - } - - return -1; -} - static \ int uncompress_zlib(struct flb_opentelemetry *ctx, char **output_buffer, @@ -352,306 +218,6 @@ static int http_header_lookup(int version, void *ptr, char *key, return -1; } -/* Used for HTTP/1.1 */ -int opentelemetry_prot_uncompress(struct flb_opentelemetry *ctx, - struct mk_http_request *request, - char **output_buffer, - size_t *output_size) -{ - int ret = 0; - char *body; - size_t body_size; - char *encoding; - size_t encoding_len; - - *output_buffer = NULL; - *output_size = 0; - - /* get the Content-Encoding */ - ret = http_header_lookup(HTTP_PROTOCOL_VERSION_11, - request, - "Content-Encoding", - &encoding, &encoding_len); - - /* FYI: no encoding was found, assume no payload compression */ - if (ret < 0) { - return 0; - } - - /* set the payload pointers */ - body = request->data.data; - body_size = request->data.len; - - if (strncasecmp(encoding, "gzip", 4) == 0 && encoding_len == 4) { - return uncompress_gzip(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "zlib", 4) == 0 && encoding_len == 4) { - return uncompress_zlib(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "zstd", 4) == 0 && encoding_len == 4) { - return uncompress_zstd(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "snappy", 6) == 0 && encoding_len == 6) { - return uncompress_snappy(ctx, - output_buffer, output_size, - body, body_size); - } - else if (strncasecmp(encoding, "deflate", 7) == 0 && encoding_len == 7) { - return uncompress_deflate(ctx, - output_buffer, output_size, - body, body_size); - } - else { - return -2; - } - - return 0; -} - - -/* - * Handle an incoming request. It performs extra checks over the request, if - * everything is OK, it enqueue the incoming payload. - */ -int opentelemetry_prot_handle(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - int i; - int ret = -1; - int len; - char *uri; - char *qs; - char *out_chunked = NULL; - flb_sds_t content_type = NULL; - size_t out_chunked_size = 0; - off_t diff; - size_t tag_len; - flb_sds_t tag; - char *original_data = NULL; - size_t original_data_size; - char *uncompressed_data = NULL; - size_t uncompressed_data_size; - struct mk_http_header *header; - - if (request->uri.data[0] != '/') { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - - /* Decode URI */ - uri = mk_utils_url_decode(request->uri); - if (!uri) { - uri = mk_mem_alloc_z(request->uri.len + 1); - if (!uri) { - return -1; - } - memcpy(uri, request->uri.data, request->uri.len); - uri[request->uri.len] = '\0'; - } - - if (strcmp(uri, "/v1/metrics") != 0 && - strcmp(uri, "/v1/traces") != 0 && - strcmp(uri, "/v1/logs") != 0) { - - send_response(conn, 400, "error: invalid endpoint\n"); - mk_mem_free(uri); - - return -1; - } - - /* Try to match a query string, so we can remove it */ - qs = strchr(uri, '?'); - if (qs) { - /* remove the query string part */ - diff = qs - uri; - uri[diff] = '\0'; - } - - /* Compose the query string using the URI */ - len = strlen(uri); - - if (ctx->tag_from_uri != FLB_TRUE) { - tag = flb_sds_create(ctx->ins->tag); - } - else { - tag = flb_sds_create_size(len); - if (!tag) { - mk_mem_free(uri); - return -1; - } - - /* New tag skipping the URI '/' */ - flb_sds_cat_safe(&tag, uri + 1, len - 1); - - /* Sanitize, only allow alphanum chars */ - for (i = 0; i < flb_sds_len(tag); i++) { - if (!isalnum(tag[i]) && tag[i] != '_' && tag[i] != '.') { - tag[i] = '_'; - } - } - } - - tag_len = flb_sds_len(tag); - - /* Check if we have a Host header: Hostname ; port */ - mk_http_point_header(&request->host, &session->parser, MK_HEADER_HOST); - - /* Header: Connection */ - mk_http_point_header(&request->connection, &session->parser, - MK_HEADER_CONNECTION); - - /* HTTP/1.1 needs Host header */ - if (!request->host.data && request->protocol == MK_HTTP_PROTOCOL_11) { - flb_sds_destroy(tag); - mk_mem_free(uri); - return -1; - } - - /* Should we close the session after this request ? */ - mk_http_keepalive_check(session, request, ctx->server); - - /* Content Length */ - header = &session->parser.headers[MK_HEADER_CONTENT_LENGTH]; - if (header->type == MK_HEADER_CONTENT_LENGTH) { - request->_content_length.data = header->val.data; - request->_content_length.len = header->val.len; - } - else { - request->_content_length.data = NULL; - } - - mk_http_point_header(&request->content_type, &session->parser, MK_HEADER_CONTENT_TYPE); - - if (request->method != MK_METHOD_POST) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(conn, 400, "error: invalid HTTP method\n"); - return -1; - } - - original_data = request->data.data; - original_data_size = request->data.len; - - if (request->data.len <= 0 && !mk_http_parser_is_content_chunked(&session->parser)) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(conn, 400, "error: no payload found\n"); - return -1; - } - - /* check if the request comes with chunked transfer encoding */ - if (mk_http_parser_is_content_chunked(&session->parser)) { - out_chunked = NULL; - out_chunked_size = 0; - - /* decode the chunks */ - ret = mk_http_parser_chunked_decode(&session->parser, - conn->buf_data, - conn->buf_len, - &out_chunked, - &out_chunked_size); - if (ret == -1) { - flb_sds_destroy(tag); - mk_mem_free(uri); - send_response(conn, 400, "error: invalid chunked data\n"); - return -1; - } - else { - request->data.data = out_chunked; - request->data.len = out_chunked_size; - } - } - - /* - * HTTP/1.x can have the payload compressed, we try to detect based on the - * Content-Encoding header. - * - * Note that HTTP/1.x real payload can only be a JSON or a Protobuf message (no gRPC) - */ - ret = opentelemetry_prot_uncompress(ctx, - request, - &uncompressed_data, - &uncompressed_data_size); - - if (ret < 0) { - flb_sds_destroy(tag); - mk_mem_free(uri); - if (out_chunked != NULL) { - mk_mem_free(out_chunked); - } - send_response(conn, 400, "error: decompression error\n"); - return -1; - } - else if (ret > 0) { - request->data.data = uncompressed_data; - request->data.len = uncompressed_data_size; - } - - if (request->content_type.data != NULL) { - content_type = flb_sds_create_len(request->content_type.data, - request->content_type.len); - } - - if (strcmp(uri, "/v1/metrics") == 0) { - ret = process_payload_metrics(ctx, conn, tag, tag_len, content_type, - session, request); - } - else if (strcmp(uri, "/v1/traces") == 0) { - ret = opentelemetry_process_traces(ctx, content_type, tag, tag_len, - request->data.data, request->data.len); - } - else if (strcmp(uri, "/v1/logs") == 0) { - ret = opentelemetry_process_logs(ctx, content_type, tag, tag_len, - request->data.data, request->data.len); - } - - request->data.data = original_data; - request->data.len = original_data_size; - - if (content_type != NULL) { - flb_sds_destroy(content_type); - } - - if (uncompressed_data != NULL) { - flb_free(uncompressed_data); - } - - if (out_chunked != NULL) { - mk_mem_free(out_chunked); - } - - mk_mem_free(uri); - flb_sds_destroy(tag); - - if (ret == -1) { - send_response(conn, 400, "error: invalid request\n"); - return -1; - } - else { - send_response(conn, ctx->successful_response_code, NULL); - } - - return ret; -} - -/* - * Handle an incoming request which has resulted in an http parser error. - */ -int opentelemetry_prot_handle_error(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request) -{ - send_response(conn, 400, "error: invalid request\n"); - return -1; -} - /* New gen HTTP server */ static int send_response_ng(struct flb_http_response *response, int http_status, diff --git a/plugins/in_opentelemetry/opentelemetry_prot.h b/plugins/in_opentelemetry/opentelemetry_prot.h index db8c6e1f38d..294e4512f90 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.h +++ b/plugins/in_opentelemetry/opentelemetry_prot.h @@ -22,15 +22,6 @@ #include -int opentelemetry_prot_handle(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - -int opentelemetry_prot_handle_error(struct flb_opentelemetry *ctx, struct http_conn *conn, - struct mk_http_session *session, - struct mk_http_request *request); - - int opentelemetry_prot_handle_ng(struct flb_http_request *request, struct flb_http_response *response); From 5689976187795dabcefe195b2e83ad267994e849 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:25:28 -0600 Subject: [PATCH 09/42] out_prometheus_exporter: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/out_prometheus_exporter/prom.c | 5 +- plugins/out_prometheus_exporter/prom_http.c | 275 +++++++------------- plugins/out_prometheus_exporter/prom_http.h | 21 +- 3 files changed, 99 insertions(+), 202 deletions(-) diff --git a/plugins/out_prometheus_exporter/prom.c b/plugins/out_prometheus_exporter/prom.c index a97a73f1cbf..9e79e3e0f03 100644 --- a/plugins/out_prometheus_exporter/prom.c +++ b/plugins/out_prometheus_exporter/prom.c @@ -89,8 +89,7 @@ static int cb_prom_init(struct flb_output_instance *ins, } /* HTTP Server context */ - ctx->http = prom_http_server_create(ctx, - ins->host.name, ins->host.port, config); + ctx->http = prom_http_server_create(ctx, config); if (!ctx->http) { flb_plg_error(ctx->ins, "could not initialize HTTP server, aborting"); return -1; @@ -311,7 +310,7 @@ struct flb_output_plugin out_prometheus_exporter_plugin = { .cb_init = cb_prom_init, .cb_flush = cb_prom_flush, .cb_exit = cb_prom_exit, - .flags = FLB_OUTPUT_NET, + .flags = FLB_OUTPUT_NET | FLB_OUTPUT_HTTP_SERVER, .event_type = FLB_OUTPUT_METRICS, .config_map = config_map, }; diff --git a/plugins/out_prometheus_exporter/prom_http.c b/plugins/out_prometheus_exporter/prom_http.c index 0298b697bba..eed6cc76662 100644 --- a/plugins/out_prometheus_exporter/prom_http.c +++ b/plugins/out_prometheus_exporter/prom_http.c @@ -19,220 +19,107 @@ #include #include +#include #include "prom.h" #include "prom_http.h" -pthread_key_t ph_metrics_key; - -/* Return the newest storage metrics buffer */ -static struct prom_http_buf *metrics_get_latest() +static int prom_http_request_handler(struct flb_http_request *request, + struct flb_http_response *response) { - struct prom_http_buf *buf; - struct mk_list *metrics_list; + cfl_sds_t payload; + struct prom_exporter *ctx; + struct prom_http *ph; - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - return NULL; + ctx = response->stream->user_data; + if (ctx == NULL || ctx->http == NULL) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } - if (mk_list_size(metrics_list) == 0) { - return NULL; - } - - buf = mk_list_entry_last(metrics_list, struct prom_http_buf, _head); - return buf; -} - -/* Delete unused metrics, note that we only care about the latest node */ -static int cleanup_metrics() -{ - int c = 0; - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list; - struct prom_http_buf *last; - struct prom_http_buf *entry; + ph = ctx->http; - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - return -1; + if (strcmp(request->path, "/") == 0) { + return flb_hs_response_send_string(response, + 200, + FLB_HS_CONTENT_TYPE_OTHER, + "Fluent Bit Prometheus Exporter\n"); } - last = metrics_get_latest(); - if (!last) { - return -1; + if (strcmp(request->path, "/metrics") != 0) { + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct prom_http_buf, _head); - if (entry != last && entry->users == 0) { - mk_list_del(&entry->_head); - flb_free(entry->buf_data); - flb_free(entry); - c++; - } + pthread_mutex_lock(&ph->metrics_mutex); + payload = ph->metrics_payload; + if (payload != NULL) { + payload = cfl_sds_create_len(payload, cfl_sds_len(payload)); } + pthread_mutex_unlock(&ph->metrics_mutex); - return c; -} - -/* destructor callback */ -static void destruct_metrics(void *data) -{ - struct mk_list *tmp; - struct mk_list *head; - struct mk_list *metrics_list = (struct mk_list*)data; - struct prom_http_buf *entry; - - if (!metrics_list) { - return; + if (payload == NULL) { + flb_http_response_set_status(response, 404); + return flb_http_response_commit(response); } + flb_hs_response_set_payload(response, + 200, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + payload, + cfl_sds_len(payload)); - mk_list_foreach_safe(head, tmp, metrics_list) { - entry = mk_list_entry(head, struct prom_http_buf, _head); - mk_list_del(&entry->_head); - flb_free(entry->buf_data); - flb_free(entry); - } + cfl_sds_destroy(payload); - flb_free(metrics_list); -} - -/* - * Callback invoked every time a new payload of Metrics is received from - * Fluent Bit engine through Message Queue channel. - * - * This function runs in a Monkey HTTP thread worker and it purpose is - * to take the metrics data and store it locally for every thread, so then - * it can be available on 'cb_metrics()' to serve it as a response. - */ -static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) -{ - struct prom_http_buf *buf; - struct mk_list *metrics_list = NULL; - - metrics_list = pthread_getspecific(ph_metrics_key); - if (!metrics_list) { - metrics_list = flb_malloc(sizeof(struct mk_list)); - if (!metrics_list) { - flb_errno(); - return; - } - mk_list_init(metrics_list); - pthread_setspecific(ph_metrics_key, metrics_list); - } - - /* FIXME: convert data ? */ - buf = flb_malloc(sizeof(struct prom_http_buf)); - if (!buf) { - flb_errno(); - return; - } - buf->users = 0; - buf->buf_data = flb_malloc(size); - if (!buf->buf_data) { - flb_errno(); - flb_free(buf); - return; - } - memcpy(buf->buf_data, data, size); - buf->buf_size = size; - - mk_list_add(&buf->_head, metrics_list); - cleanup_metrics(); -} - -/* Create message queue to receive Metrics payload from the engine */ -static int http_server_mq_create(struct prom_http *ph) -{ - int ret; - - pthread_key_create(&ph_metrics_key, destruct_metrics); - - ret = mk_mq_create(ph->ctx, "/metrics", cb_mq_metrics, NULL); - if (ret == -1) { - return -1; - } - ph->qid_metrics = ret; return 0; } -/* HTTP endpoint: /metrics */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct prom_http_buf *buf; - (void) data; - - buf = metrics_get_latest(); - if (!buf) { - mk_http_status(request, 404); - mk_http_done(request); - return; - } - - buf->users++; - - mk_http_status(request, 200); - flb_hs_add_content_type_to_req(request, FLB_HS_CONTENT_TYPE_PROMETHEUS); - mk_http_send(request, buf->buf_data, buf->buf_size, NULL); - mk_http_done(request); - - buf->users--; -} - -/* HTTP endpoint: / (root) */ -static void cb_root(mk_request_t *request, void *data) -{ - (void) data; - - mk_http_status(request, 200); - mk_http_send(request, "Fluent Bit Prometheus Exporter\n", 31, NULL); - mk_http_done(request); -} - struct prom_http *prom_http_server_create(struct prom_exporter *ctx, - const char *listen, - int tcp_port, struct flb_config *config) { int ret; - int vid; - char tmp[32]; + int protocol_version; struct prom_http *ph; + struct flb_output_instance *ins; + struct flb_http_server_options options; - ph = flb_malloc(sizeof(struct prom_http)); + ph = flb_calloc(1, sizeof(struct prom_http)); if (!ph) { flb_errno(); return NULL; } + + ins = ctx->ins; ph->config = config; + pthread_mutex_init(&ph->metrics_mutex, NULL); - /* HTTP Server context */ - ph->ctx = mk_create(); - if (!ph->ctx) { - flb_free(ph); - return NULL; + if (ins->http_server_config != NULL && + ins->http_server_config->http2 == FLB_FALSE) { + protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else { + protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; } - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%d", listen, tcp_port); - mk_config_set(ph->ctx, - "Listen", tmp, - "Workers", "1", - NULL); - - /* Virtual host */ - vid = mk_vhost_create(ph->ctx, NULL); - ph->vid = vid; + flb_http_server_options_init(&options); + options.protocol_version = protocol_version; + options.flags = 0; + options.request_callback = prom_http_request_handler; + options.user_data = ctx; + options.address = ins->host.name; + options.port = ins->host.port; + options.networking_flags = ins->flags; + options.networking_setup = &ins->net_setup; + options.event_loop = config->evl; + options.system_context = config; + options.use_caller_event_loop = FLB_TRUE; - /* Set HTTP URI callbacks */ - mk_vhost_handler(ph->ctx, vid, "/metrics", cb_metrics, NULL); - mk_vhost_handler(ph->ctx, vid, "/", cb_root, NULL); + if (ins->http_server_config != NULL) { + options.buffer_max_size = ins->http_server_config->buffer_max_size; + options.max_connections = ins->http_server_config->max_connections; + } - /* Create a Message Queue to push 'metrics' to HTTP workers */ - ret = http_server_mq_create(ph); - if (ret == -1) { - mk_destroy(ph->ctx); + ret = flb_http_server_init_with_options(&ph->server, &options); + if (ret != 0) { + pthread_mutex_destroy(&ph->metrics_mutex); flb_free(ph); return NULL; } @@ -243,26 +130,46 @@ struct prom_http *prom_http_server_create(struct prom_exporter *ctx, void prom_http_server_destroy(struct prom_http *ph) { if (ph) { - /* TODO: release mk_vhost */ - if (ph->ctx) { - mk_destroy(ph->ctx); + flb_http_server_destroy(&ph->server); + + if (ph->metrics_payload != NULL) { + cfl_sds_destroy(ph->metrics_payload); } + + pthread_mutex_destroy(&ph->metrics_mutex); flb_free(ph); } } int prom_http_server_start(struct prom_http *ph) { - return mk_start(ph->ctx); + return flb_http_server_start(&ph->server); } int prom_http_server_stop(struct prom_http *ph) { - return mk_stop(ph->ctx); + return flb_http_server_stop(&ph->server); } int prom_http_server_mq_push_metrics(struct prom_http *ph, void *data, size_t size) { - return mk_mq_send(ph->ctx, ph->qid_metrics, data, size); + cfl_sds_t new_payload; + + new_payload = cfl_sds_create_len(data, size); + if (new_payload == NULL) { + return -1; + } + + pthread_mutex_lock(&ph->metrics_mutex); + + if (ph->metrics_payload != NULL) { + cfl_sds_destroy(ph->metrics_payload); + } + + ph->metrics_payload = new_payload; + + pthread_mutex_unlock(&ph->metrics_mutex); + + return 0; } diff --git a/plugins/out_prometheus_exporter/prom_http.h b/plugins/out_prometheus_exporter/prom_http.h index 572742de5f8..06cd7b132f9 100644 --- a/plugins/out_prometheus_exporter/prom_http.h +++ b/plugins/out_prometheus_exporter/prom_http.h @@ -21,29 +21,20 @@ #define FLB_PROMETHEUS_EXPORTER_HTTP_H #include -#include +#include +#include #include "prom.h" -/* HTTP response payload received through a Message Queue */ -struct prom_http_buf { - int users; - char *buf_data; - size_t buf_size; - struct mk_list _head; -}; - /* Prom HTTP Server context */ struct prom_http { - mk_ctx_t *ctx; /* Monkey HTTP Context */ - int vid; /* Virtual host ID */ - int qid_metrics; /* Queue ID for Metrics buffer */ - struct flb_config *config; /* Fluent Bit context */ + struct flb_http_server server; + pthread_mutex_t metrics_mutex; + cfl_sds_t metrics_payload; + struct flb_config *config; }; struct prom_http *prom_http_server_create(struct prom_exporter *ctx, - const char *listen, - int tcp_port, struct flb_config *config); void prom_http_server_destroy(struct prom_http *ph); From 8ebe51cf0124eb47ce858b1c2383138fa3f202bf Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:25:39 -0600 Subject: [PATCH 10/42] out_vivo_exporter: unify HTTP server path Signed-off-by: Eduardo Silva --- plugins/out_vivo_exporter/vivo.c | 11 +- plugins/out_vivo_exporter/vivo_http.c | 375 +++++++++++++++----------- plugins/out_vivo_exporter/vivo_http.h | 20 +- 3 files changed, 218 insertions(+), 188 deletions(-) diff --git a/plugins/out_vivo_exporter/vivo.c b/plugins/out_vivo_exporter/vivo.c index 6fd94b1b268..64b263d2267 100644 --- a/plugins/out_vivo_exporter/vivo.c +++ b/plugins/out_vivo_exporter/vivo.c @@ -373,12 +373,6 @@ static int cb_vivo_init(struct flb_output_instance *ins, ctx->ins = ins; ctx->config = config; - ret = flb_output_config_map_set(ins, (void *) ctx); - if (ret == -1) { - flb_free(ctx); - return -1; - } - flb_output_set_context(ins, ctx); /* Load config map */ @@ -404,8 +398,7 @@ static int cb_vivo_init(struct flb_output_instance *ins, } /* HTTP Server context */ - ctx->http = vivo_http_server_create(ctx, - ins->host.name, ins->host.port, config); + ctx->http = vivo_http_server_create(ctx, config); if (!ctx->http) { flb_plg_error(ctx->ins, "could not initialize HTTP server, aborting"); return -1; @@ -505,7 +498,7 @@ struct flb_output_plugin out_vivo_exporter_plugin = { .cb_init = cb_vivo_init, .cb_flush = cb_vivo_flush, .cb_exit = cb_vivo_exit, - .flags = FLB_OUTPUT_NET, + .flags = FLB_OUTPUT_NET | FLB_OUTPUT_HTTP_SERVER, .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES, .config_map = config_map, .workers = 1, diff --git a/plugins/out_vivo_exporter/vivo_http.c b/plugins/out_vivo_exporter/vivo_http.c index de36540545d..de523e04307 100644 --- a/plugins/out_vivo_exporter/vivo_http.c +++ b/plugins/out_vivo_exporter/vivo_http.c @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -29,10 +30,15 @@ #include "vivo_http.h" #include "vivo_stream.h" -#define VIVO_CONTENT_TYPE "Content-Type" -#define VIVO_CONTENT_TYPE_JSON "application/json" -static int stream_get_uri_properties(mk_request_t *request, - int64_t *from, int64_t *to, int64_t *limit) +#define VIVO_ACCESS_CONTROL_ALLOW_HEADERS_VALUE \ + "Origin, X-Requested-With, Content-Type, Accept" +#define VIVO_ACCESS_CONTROL_EXPOSE_HEADERS_VALUE \ + "vivo-stream-start-id, vivo-stream-end-id, vivo-stream-next-id" + +static int stream_get_query_properties(struct flb_http_request *request, + int64_t *from, + int64_t *to, + int64_t *limit) { char *ptr; flb_sds_t buf; @@ -41,7 +47,11 @@ static int stream_get_uri_properties(mk_request_t *request, *to = -1; *limit = -1; - buf = flb_sds_create_len(request->query_string.data, request->query_string.len); + if (request->query_string == NULL) { + return 0; + } + + buf = flb_sds_create_len(request->query_string, cfl_sds_len(request->query_string)); if (!buf) { return -1; } @@ -66,175 +76,171 @@ static int stream_get_uri_properties(mk_request_t *request, return 0; } -static void headers_set_common(struct vivo_exporter *ctx, mk_request_t *request) +static int headers_set_common(struct flb_http_response *response, + struct vivo_exporter *ctx) { - /* content type */ - mk_http_header(request, - VIVO_CONTENT_TYPE, sizeof(VIVO_CONTENT_TYPE) - 1, - VIVO_CONTENT_TYPE_JSON, sizeof(VIVO_CONTENT_TYPE_JSON) - 1); - - /* CORS */ - if (ctx->http_cors_allow_origin) { - mk_http_header(request, - "Access-Control-Allow-Origin", - sizeof("Access-Control-Allow-Origin") - 1, - ctx->http_cors_allow_origin, - flb_sds_len(ctx->http_cors_allow_origin)); - - mk_http_header(request, - "Access-Control-Allow-Headers", - sizeof("Access-Control-Allow-Headers") - 1, - "Origin, X-Requested-With, Content-Type, Accept", - sizeof("Origin, X-Requested-With, Content-Type, Accept") - 1); + flb_hs_response_set_content_type(response, FLB_HS_CONTENT_TYPE_JSON); + + if (ctx->http_cors_allow_origin != NULL) { + flb_http_response_set_header( + response, + "Access-Control-Allow-Origin", + sizeof("Access-Control-Allow-Origin") - 1, + ctx->http_cors_allow_origin, + flb_sds_len(ctx->http_cors_allow_origin)); + + flb_http_response_set_header( + response, + "Access-Control-Allow-Headers", + sizeof("Access-Control-Allow-Headers") - 1, + VIVO_ACCESS_CONTROL_ALLOW_HEADERS_VALUE, + sizeof(VIVO_ACCESS_CONTROL_ALLOW_HEADERS_VALUE) - 1); } + + return 0; } -static void headers_set(mk_request_t *request, struct vivo_stream *vs) +static int headers_set(struct flb_http_response *response, struct vivo_stream *vs) { struct vivo_exporter *ctx; - - /* parent context */ ctx = vs->parent; - - headers_set_common(ctx, request); - - if (ctx->http_cors_allow_origin) { - mk_http_header(request, - "Access-Control-Expose-Headers", - sizeof("Access-Control-Expose-Headers") - 1, - "vivo-stream-start-id, vivo-stream-end-id, vivo-stream-next-id", - sizeof("vivo-stream-start-id, vivo-stream-end-id, vivo-stream-next-id") - 1); + headers_set_common(response, ctx); + + if (ctx->http_cors_allow_origin != NULL) { + flb_http_response_set_header( + response, + "Access-Control-Expose-Headers", + sizeof("Access-Control-Expose-Headers") - 1, + VIVO_ACCESS_CONTROL_EXPOSE_HEADERS_VALUE, + sizeof(VIVO_ACCESS_CONTROL_EXPOSE_HEADERS_VALUE) - 1); } + + return 0; } -void vivo_http_serve_content(mk_request_t *request, struct vivo_stream *vs) +static int vivo_http_serve_content(struct flb_http_request *request, + struct flb_http_response *response, + struct vivo_stream *vs) { - int64_t from = -1; - int64_t to = -1; - int64_t limit = -1; - int64_t stream_start_id = -1; - int64_t stream_end_id = -1; - int64_t stream_next_id = -1; + int64_t from; + int64_t to; + int64_t limit; + int64_t stream_start_id; + int64_t stream_end_id; + int64_t stream_next_id; flb_sds_t payload; flb_sds_t str_start; flb_sds_t str_end; flb_sds_t str_next; - - if (request->query_string.len > 0) { - stream_get_uri_properties(request, &from, &to, &limit); + if (stream_get_query_properties(request, &from, &to, &limit) != 0) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } payload = vivo_stream_get_content(vs, from, to, limit, &stream_start_id, &stream_end_id, &stream_next_id); if (!payload) { - mk_http_status(request, 500); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } - mk_http_status(request, 200); - - /* set response headers */ - headers_set(request, vs); + flb_http_response_set_status(response, 200); + headers_set(response, vs); str_next = flb_sds_create_size(32); - flb_sds_printf(&str_next, "%" PRId64, stream_next_id); + if (str_next == NULL) { + flb_sds_destroy(payload); + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } - mk_http_header(request, - VIVO_STREAM_NEXT_ID, sizeof(VIVO_STREAM_NEXT_ID) - 1, - str_next, flb_sds_len(str_next)); + flb_sds_printf(&str_next, "%" PRId64, stream_next_id); + flb_http_response_set_header(response, + VIVO_STREAM_NEXT_ID, + sizeof(VIVO_STREAM_NEXT_ID) - 1, + str_next, + flb_sds_len(str_next)); if (flb_sds_len(payload) == 0) { flb_sds_destroy(payload); flb_sds_destroy(str_next); - return; + flb_http_response_set_body(response, NULL, 0); + return flb_http_response_commit(response); } - /* stream ids served: compose buffer and set headers */ str_start = flb_sds_create_size(32); - flb_sds_printf(&str_start, "%" PRId64, stream_start_id); - str_end = flb_sds_create_size(32); - flb_sds_printf(&str_end, "%" PRId64, stream_end_id); - - mk_http_header(request, - VIVO_STREAM_START_ID, sizeof(VIVO_STREAM_START_ID) - 1, - str_start, flb_sds_len(str_start)); - - mk_http_header(request, - VIVO_STREAM_END_ID, sizeof(VIVO_STREAM_END_ID) - 1, - str_end, flb_sds_len(str_end)); - /* send payload */ - mk_http_send(request, payload, flb_sds_len(payload), NULL); - - /* release */ - flb_sds_destroy(payload); - flb_sds_destroy(str_start); - flb_sds_destroy(str_end); - flb_sds_destroy(str_next); -} + if (str_start == NULL || str_end == NULL) { + flb_sds_destroy(payload); + flb_sds_destroy(str_next); -/* HTTP endpoint: /api/v1/logs */ -static void cb_logs(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; + if (str_start != NULL) { + flb_sds_destroy(str_start); + } - ctx = (struct vivo_exporter *) data; + if (str_end != NULL) { + flb_sds_destroy(str_end); + } - vivo_http_serve_content(request, ctx->stream_logs); - mk_http_done(request); -} + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } -/* HTTP endpoint: /api/v1/metrics */ -static void cb_metrics(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; + flb_sds_printf(&str_start, "%" PRId64, stream_start_id); + flb_sds_printf(&str_end, "%" PRId64, stream_end_id); - ctx = (struct vivo_exporter *) data; + flb_http_response_set_header(response, + VIVO_STREAM_START_ID, + sizeof(VIVO_STREAM_START_ID) - 1, + str_start, + flb_sds_len(str_start)); - vivo_http_serve_content(request, ctx->stream_metrics); - mk_http_done(request); -} + flb_http_response_set_header(response, + VIVO_STREAM_END_ID, + sizeof(VIVO_STREAM_END_ID) - 1, + str_end, + flb_sds_len(str_end)); -/* HTTP endpoint: /api/v1/traces */ -static void cb_traces(mk_request_t *request, void *data) -{ - struct vivo_exporter *ctx; + flb_http_response_set_body(response, + (unsigned char *) payload, + flb_sds_len(payload)); - ctx = (struct vivo_exporter *) data; + flb_sds_destroy(payload); + flb_sds_destroy(str_start); + flb_sds_destroy(str_end); + flb_sds_destroy(str_next); - vivo_http_serve_content(request, ctx->stream_traces); - mk_http_done(request); + return flb_http_response_commit(response); } -/* HTTP endpoint: /api/v1/internal/metrics */ -static void cb_internal_metrics(mk_request_t *request, void *data) +static int cb_internal_metrics(struct flb_http_response *response, + struct vivo_exporter *ctx) { int ret; - char *mp_buf = NULL; - size_t mp_size = 0; - flb_sds_t json = NULL; - struct cmt *cmt = NULL; - struct vivo_exporter *ctx; + char *mp_buf; + size_t mp_size; + flb_sds_t json; + struct cmt *cmt; - ctx = (struct vivo_exporter *) data; + mp_buf = NULL; + mp_size = 0; + json = NULL; cmt = flb_me_get_cmetrics(ctx->config); if (!cmt) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } ret = cmt_encode_msgpack_create(cmt, &mp_buf, &mp_size); if (ret != 0) { cmt_destroy(cmt); - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } json = flb_msgpack_raw_to_json_sds(mp_buf, mp_size, @@ -244,90 +250,135 @@ static void cb_internal_metrics(mk_request_t *request, void *data) cmt_destroy(cmt); if (!json) { - mk_http_status(request, 500); - mk_http_done(request); - return; + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); } - mk_http_status(request, 200); - headers_set_common(ctx, request); - mk_http_send(request, json, flb_sds_len(json), NULL); - mk_http_done(request); - + flb_http_response_set_status(response, 200); + headers_set_common(response, ctx); + flb_http_response_set_body(response, + (unsigned char *) json, + flb_sds_len(json)); flb_sds_destroy(json); + + return flb_http_response_commit(response); } -/* HTTP endpoint: / (root) */ -static void cb_root(mk_request_t *request, void *data) +static int vivo_http_request_handler(struct flb_http_request *request, + struct flb_http_response *response) { - (void) data; + struct vivo_exporter *ctx; - mk_http_status(request, 200); - mk_http_send(request, "Fluent Bit Vivo Exporter\n", 24, NULL); - mk_http_done(request); + ctx = response->stream->user_data; + if (ctx == NULL) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } + + if (strcmp(request->path, "/api/v1/logs") == 0) { + return vivo_http_serve_content(request, response, ctx->stream_logs); + } + + if (strcmp(request->path, "/api/v1/metrics") == 0) { + return vivo_http_serve_content(request, response, ctx->stream_metrics); + } + + if (strcmp(request->path, "/api/v1/traces") == 0) { + return vivo_http_serve_content(request, response, ctx->stream_traces); + } + + if (strcmp(request->path, "/api/v1/internal/metrics") == 0) { + return cb_internal_metrics(response, ctx); + } + + if (strcmp(request->path, "/") == 0) { + return flb_hs_response_send_string(response, + 200, + FLB_HS_CONTENT_TYPE_OTHER, + "Fluent Bit Vivo Exporter\n"); + } + + flb_http_response_set_status(response, 404); + + return flb_http_response_commit(response); } struct vivo_http *vivo_http_server_create(struct vivo_exporter *ctx, - const char *listen, - int tcp_port, struct flb_config *config) { - int vid; - char tmp[32]; + int ret; + int protocol_version; struct vivo_http *ph; + struct flb_output_instance *ins; + struct flb_http_server_options options; - ph = flb_malloc(sizeof(struct vivo_http)); + ph = flb_calloc(1, sizeof(struct vivo_http)); if (!ph) { flb_errno(); return NULL; } + ph->config = config; + ins = ctx->ins; - /* HTTP Server context */ - ph->ctx = mk_create(); - if (!ph->ctx) { - flb_free(ph); - return NULL; + if (ins->http_server_config != NULL && + ins->http_server_config->http2 == FLB_FALSE) { + protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else { + protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; } - /* Compose listen address */ - snprintf(tmp, sizeof(tmp) -1, "%s:%d", listen, tcp_port); - mk_config_set(ph->ctx, - "Listen", tmp, - "Workers", "1", - NULL); - - /* Virtual host */ - vid = mk_vhost_create(ph->ctx, NULL); - ph->vid = vid; + flb_http_server_options_init(&options); + options.protocol_version = protocol_version; + options.request_callback = vivo_http_request_handler; + options.user_data = ctx; + options.address = ins->host.name; + options.port = ins->host.port; + options.networking_flags = ins->flags; + options.networking_setup = &ins->net_setup; + options.event_loop = config->evl; + options.system_context = config; + options.use_caller_event_loop = FLB_TRUE; + + if (ins->http_server_config != NULL) { + options.buffer_max_size = ins->http_server_config->buffer_max_size; + options.max_connections = ins->http_server_config->max_connections; + } - /* Set HTTP URI callbacks */ - mk_vhost_handler(ph->ctx, vid, "/api/v1/logs", cb_logs, ctx); - mk_vhost_handler(ph->ctx, vid, "/api/v1/metrics", cb_metrics, ctx); - mk_vhost_handler(ph->ctx, vid, "/api/v1/traces", cb_traces, ctx); - mk_vhost_handler(ph->ctx, vid, "/api/v1/internal/metrics", cb_internal_metrics, ctx); - mk_vhost_handler(ph->ctx, vid, "/", cb_root, NULL); + ret = flb_http_server_init_with_options(&ph->server, &options); + if (ret != 0) { + flb_free(ph); + return NULL; + } return ph; } void vivo_http_server_destroy(struct vivo_http *ph) { - if (ph) { - /* TODO: release mk_vhost */ - if (ph->ctx) { - mk_destroy(ph->ctx); - } + if (ph != NULL) { + flb_http_server_destroy(&ph->server); flb_free(ph); } } int vivo_http_server_start(struct vivo_http *ph) { - return mk_start(ph->ctx); + return flb_http_server_start(&ph->server); } int vivo_http_server_stop(struct vivo_http *ph) { - return mk_stop(ph->ctx); + return flb_http_server_stop(&ph->server); +} + +int vivo_http_server_mq_push_metrics(struct vivo_http *ph, + void *data, size_t size) +{ + (void) ph; + (void) data; + (void) size; + + return 0; } diff --git a/plugins/out_vivo_exporter/vivo_http.h b/plugins/out_vivo_exporter/vivo_http.h index 8557aa28ddc..1f1009d5447 100644 --- a/plugins/out_vivo_exporter/vivo_http.h +++ b/plugins/out_vivo_exporter/vivo_http.h @@ -21,7 +21,7 @@ #define FLB_VIVO_EXPORTER_HTTP_H #include -#include +#include #include "vivo.h" @@ -31,25 +31,13 @@ struct vivo_stream; -/* HTTP response payload received through a Message Queue */ -struct vivo_http_buf { - int users; - char *buf_data; - size_t buf_size; - struct mk_list _head; -}; - /* Vivo HTTP Server context */ struct vivo_http { - mk_ctx_t *ctx; /* Monkey HTTP Context */ - int vid; /* Virtual host ID */ - int qid_metrics; /* Queue ID for Metrics buffer */ - struct flb_config *config; /* Fluent Bit context */ + struct flb_http_server server; + struct flb_config *config; }; struct vivo_http *vivo_http_server_create(struct vivo_exporter *ctx, - const char *listen, - int tcp_port, struct flb_config *config); void vivo_http_server_destroy(struct vivo_http *ph); @@ -59,6 +47,4 @@ int vivo_http_server_stop(struct vivo_http *ph); int vivo_http_server_mq_push_metrics(struct vivo_http *ph, void *data, size_t size); -void vivo_http_serve_content(mk_request_t *request, struct vivo_stream *vs); - #endif From 1f07d9ce636a16e85eb15613da3342d06d56d828 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:25:50 -0600 Subject: [PATCH 11/42] tests: internal: add HTTP server coverage Signed-off-by: Eduardo Silva --- tests/internal/CMakeLists.txt | 7 ++ tests/internal/http_server.c | 190 ++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 tests/internal/http_server.c diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 9b5b8fdd3d7..ec84a3bcdf0 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -121,6 +121,13 @@ if(FLB_METRICS) ) endif() +if(FLB_HTTP_SERVER) + set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + http_server.c + ) +endif() + if(FLB_SIGNV4) set(UNIT_TESTS_FILES ${UNIT_TESTS_FILES} diff --git a/tests/internal/http_server.c b/tests/internal/http_server.c new file mode 100644 index 00000000000..20d425c02ce --- /dev/null +++ b/tests/internal/http_server.c @@ -0,0 +1,190 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "flb_tests_internal.h" + +#define TEST_HTTP_SERVER_HOST "127.0.0.1" + +struct test_http_server_context { + pthread_mutex_t lock; + int init_calls; + int exit_calls; + int request_calls; +}; + +static void test_http_server_context_init(struct test_http_server_context *context) +{ + memset(context, 0, sizeof(struct test_http_server_context)); + pthread_mutex_init(&context->lock, NULL); +} + +static void test_http_server_context_destroy(struct test_http_server_context *context) +{ + pthread_mutex_destroy(&context->lock); +} + +static int test_http_server_worker_init(struct flb_http_server *server, void *data) +{ + struct test_http_server_context *context; + + (void) server; + + context = data; + + pthread_mutex_lock(&context->lock); + context->init_calls++; + pthread_mutex_unlock(&context->lock); + + return 0; +} + +static int test_http_server_worker_exit(struct flb_http_server *server, void *data) +{ + struct test_http_server_context *context; + + (void) server; + + context = data; + + pthread_mutex_lock(&context->lock); + context->exit_calls++; + pthread_mutex_unlock(&context->lock); + + return 0; +} + +static int test_http_server_request_handler(struct flb_http_request *request, + struct flb_http_response *response) +{ + struct test_http_server_context *context; + struct flb_http_server_session *session; + + session = (struct flb_http_server_session *) request->stream->parent; + context = session->parent->user_data; + + pthread_mutex_lock(&context->lock); + context->request_calls++; + pthread_mutex_unlock(&context->lock); + + flb_http_response_set_status(response, 200); + flb_http_response_set_body(response, + (unsigned char *) "ok", + 2); + + return flb_http_response_commit(response); +} + +void test_http_server_options_defaults() +{ + struct flb_http_server_options options; + struct flb_http_server_config config; + + flb_http_server_options_init(&options); + flb_http_server_config_init(&config); + + TEST_CHECK(options.workers == 1); + TEST_CHECK(options.use_caller_event_loop == FLB_TRUE); + TEST_CHECK(options.reuse_port == FLB_FALSE); + TEST_CHECK(options.buffer_max_size == HTTP_SERVER_MAXIMUM_BUFFER_SIZE); + TEST_CHECK(options.max_connections == 0); + TEST_CHECK(config.http2 == FLB_TRUE); + TEST_CHECK(config.buffer_max_size == HTTP_SERVER_MAXIMUM_BUFFER_SIZE); + TEST_CHECK(config.max_connections == 0); +} + +void test_http_server_options_multi_worker_magic() +{ + struct flb_config *config; + struct flb_net_setup net_setup; + struct flb_http_server server; + struct flb_http_server_options options; + + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + return; + } + + flb_net_setup_init(&net_setup); + flb_http_server_options_init(&options); + + options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + options.request_callback = test_http_server_request_handler; + options.address = (char *) TEST_HTTP_SERVER_HOST; + options.port = 10001; + options.networking_flags = FLB_IO_TCP; + options.networking_setup = &net_setup; + options.system_context = config; + options.workers = 2; + options.max_connections = 7; + + TEST_CHECK(flb_http_server_init_with_options(&server, &options) == 0); + TEST_CHECK(server.workers == 2); + TEST_CHECK(server.reuse_port == FLB_TRUE); + TEST_CHECK(server.use_caller_event_loop == FLB_FALSE); + TEST_CHECK(net_setup.share_port == FLB_TRUE); + TEST_CHECK(server.max_connections == 7); + + flb_http_server_destroy(&server); + flb_config_exit(config); +} + +void test_http_server_managed_worker_contract() +{ + struct flb_config *config; + struct flb_net_setup net_setup; + struct flb_http_server server; + struct flb_http_server_options options; + struct test_http_server_context context; + config = flb_config_init(); + if (!TEST_CHECK(config != NULL)) { + return; + } + + test_http_server_context_init(&context); + + flb_net_setup_init(&net_setup); + flb_http_server_options_init(&options); + + options.protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + options.request_callback = test_http_server_request_handler; + options.user_data = &context; + options.address = (char *) TEST_HTTP_SERVER_HOST; + options.port = 10002; + options.networking_flags = FLB_IO_TCP; + options.networking_setup = &net_setup; + options.system_context = config; + options.workers = 2; + options.max_connections = 3; + options.cb_worker_init = test_http_server_worker_init; + options.cb_worker_exit = test_http_server_worker_exit; + + TEST_CHECK(flb_http_server_init_with_options(&server, &options) == 0); + TEST_CHECK(server.workers == 2); + TEST_CHECK(server.use_caller_event_loop == FLB_FALSE); + TEST_CHECK(server.reuse_port == FLB_TRUE); + TEST_CHECK(server.max_connections == 3); + TEST_CHECK(server.cb_worker_init == test_http_server_worker_init); + TEST_CHECK(server.cb_worker_exit == test_http_server_worker_exit); + TEST_CHECK(context.request_calls == 0); + TEST_CHECK(context.init_calls == 0); + TEST_CHECK(context.exit_calls == 0); + + test_http_server_context_destroy(&context); + flb_config_exit(config); +} + +TEST_LIST = { + { "http_server_options_defaults", test_http_server_options_defaults }, + { "http_server_options_multi_worker_magic", test_http_server_options_multi_worker_magic }, + { "http_server_managed_worker_contract", test_http_server_managed_worker_contract }, + { 0 } +}; From e274c2681c03474aa111bc87f7b7782d57850059 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 11:26:08 -0600 Subject: [PATCH 12/42] help: expose HTTP server options Signed-off-by: Eduardo Silva --- src/flb_help.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/flb_help.c b/src/flb_help.c index f109ad334b6..eda74f92f00 100644 --- a/src/flb_help.c +++ b/src/flb_help.c @@ -27,6 +27,7 @@ #include #include #include +#include @@ -269,6 +270,11 @@ int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_s if ((ins->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) != 0) { options_size += 3; } + if ((ins->flags & FLB_INPUT_HTTP_SERVER) != 0) { + config_map = flb_http_server_get_config_map(ins->config); + options_size += mk_list_size(config_map); + flb_config_map_destroy(config_map); + } msgpack_pack_array(&mp_pck, options_size); @@ -277,6 +283,14 @@ int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_s pack_config_map_entry(&mp_pck, &m_input_net_host); pack_config_map_entry(&mp_pck, &m_input_net_port); } + if ((ins->flags & FLB_INPUT_HTTP_SERVER) != 0) { + config_map = flb_http_server_get_config_map(ins->config); + mk_list_foreach(head, config_map) { + m = mk_list_entry(head, struct flb_config_map, _head); + pack_config_map_entry(&mp_pck, m); + } + flb_config_map_destroy(config_map); + } mk_list_foreach(head, config_map) { m = mk_list_entry(head, struct flb_config_map, _head); @@ -537,6 +551,11 @@ int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out if (ins->flags & FLB_OUTPUT_NET) { options_size += 2; } + if (ins->flags & FLB_OUTPUT_HTTP_SERVER) { + config_map = flb_http_server_get_config_map(ins->config); + options_size += mk_list_size(config_map); + flb_config_map_destroy(config_map); + } if (ins->flags & FLB_IO_OPT_TLS) { tls_config = flb_tls_get_config_map(ins->config); options_size += mk_list_size(tls_config); @@ -548,6 +567,14 @@ int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out pack_config_map_entry(&mp_pck, &m_output_net_host); pack_config_map_entry(&mp_pck, &m_output_net_port); } + if (ins->flags & FLB_OUTPUT_HTTP_SERVER) { + config_map = flb_http_server_get_config_map(ins->config); + mk_list_foreach(head, config_map) { + m = mk_list_entry(head, struct flb_config_map, _head); + pack_config_map_entry(&mp_pck, m); + } + flb_config_map_destroy(config_map); + } if (ins->flags & FLB_IO_OPT_TLS) { mk_list_foreach(head, tls_config) { m = mk_list_entry(head, struct flb_config_map, _head); From 59d1da7194b7eb3fb5ccf15ac68bdf6811e65e1f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 13:07:47 -0600 Subject: [PATCH 13/42] http_server: fix ALPN fallback for HTTP/1 TLS listeners Signed-off-by: Eduardo Silva --- src/http_server/flb_http_server.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index cc811d8c4cd..6059487e57c 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -78,6 +78,27 @@ static int flb_http_server_running_on_caller_context( session->event_loop != NULL; } +static const char *flb_http_server_get_alpn_string(struct flb_http_server *session) +{ + if (session == NULL) { + return NULL; + } + + if (session->protocol_version == HTTP_PROTOCOL_VERSION_AUTODETECT) { + return "h2,http/1.1,http/1.0"; + } + + if (session->protocol_version >= HTTP_PROTOCOL_VERSION_20) { + return "h2"; + } + + if (session->protocol_version == HTTP_PROTOCOL_VERSION_11) { + return "http/1.1,http/1.0"; + } + + return "http/1.0"; +} + static size_t flb_http_server_client_count(struct flb_http_server *server) { return cfl_list_size(&server->clients); @@ -805,6 +826,7 @@ int flb_http_server_init_with_options( int flb_http_server_start(struct flb_http_server *session) { + const char *alpn; int result; if (!flb_http_server_running_on_caller_context(session)) { @@ -813,7 +835,8 @@ int flb_http_server_start(struct flb_http_server *session) } if (session->tls_provider != NULL) { - result = flb_tls_set_alpn(session->tls_provider, "h2,http/1.0,http/1.1"); + alpn = flb_http_server_get_alpn_string(session); + result = flb_tls_set_alpn(session->tls_provider, alpn); if (result != 0) { return -1; From f8000fb7c45adc012f31066db0c3c1bba5151b12 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:21:47 -0600 Subject: [PATCH 14/42] in_http: accept parameterized form content types Signed-off-by: Eduardo Silva --- plugins/in_http/http_prot.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/plugins/in_http/http_prot.c b/plugins/in_http/http_prot.c index b9fd0efe037..e4b3ca70a9a 100644 --- a/plugins/in_http/http_prot.c +++ b/plugins/in_http/http_prot.c @@ -65,12 +65,33 @@ static int content_type_is_json(const char *content_type) static int content_type_is_urlencoded(const char *content_type) { + const char *separator; + size_t media_type_length; + if (content_type == NULL) { return FLB_FALSE; } - return strcasecmp(content_type, - "application/x-www-form-urlencoded") == 0; + separator = strchr(content_type, ';'); + if (separator == NULL) { + return strcasecmp(content_type, + "application/x-www-form-urlencoded") == 0; + } + + media_type_length = separator - content_type; + + while (media_type_length > 0 && + isspace((unsigned char) content_type[media_type_length - 1])) { + media_type_length--; + } + + if (media_type_length != strlen("application/x-www-form-urlencoded")) { + return FLB_FALSE; + } + + return strncasecmp(content_type, + "application/x-www-form-urlencoded", + media_type_length) == 0; } static inline char hex2nibble(char c) From 213d48139363262bac9439a405a9fe8d0356faa4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:21:58 -0600 Subject: [PATCH 15/42] in_prometheus_remote_write: fix ingest error handling Signed-off-by: Eduardo Silva --- plugins/in_prometheus_remote_write/prom_rw.c | 4 ++-- .../in_prometheus_remote_write/prom_rw_config.c | 4 ++++ .../in_prometheus_remote_write/prom_rw_prot.c | 16 ++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/plugins/in_prometheus_remote_write/prom_rw.c b/plugins/in_prometheus_remote_write/prom_rw.c index df96f17a2fa..69508734ac2 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.c +++ b/plugins/in_prometheus_remote_write/prom_rw.c @@ -4,11 +4,11 @@ * ========== * Copyright (C) 2015-2026 The Fluent Bit Authors * - * Licensed under the Apache License, Version 2.in_in (the "License"); + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.in_in + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/in_prometheus_remote_write/prom_rw_config.c b/plugins/in_prometheus_remote_write/prom_rw_config.c index c6474bfa93d..20d6dbc38e1 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_config.c +++ b/plugins/in_prometheus_remote_write/prom_rw_config.c @@ -57,6 +57,10 @@ struct flb_prom_remote_write *prom_rw_config_create(struct flb_input_instance *i int prom_rw_config_destroy(struct flb_prom_remote_write *ctx) { + if (ctx == NULL) { + return 0; + } + flb_http_server_destroy(&ctx->http_server); flb_free(ctx->listen); diff --git a/plugins/in_prometheus_remote_write/prom_rw_prot.c b/plugins/in_prometheus_remote_write/prom_rw_prot.c index 96e4f5543b3..fe55791f50f 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_prot.c +++ b/plugins/in_prometheus_remote_write/prom_rw_prot.c @@ -42,6 +42,9 @@ static int send_response_ng(struct flb_http_response *response, else if (http_status == 400) { flb_http_response_set_message(response, "Bad Request"); } + else if (http_status == 500) { + flb_http_response_set_message(response, "Internal Server Error"); + } if (message != NULL) { flb_http_response_set_body(response, @@ -65,15 +68,15 @@ static int process_payload_metrics_ng(struct flb_prom_remote_write *ctx, cfl_sds_len(request->body)); if (result != CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - return -1; + return 400; } result = flb_input_metrics_append(ctx->ins, NULL, 0, context); cmt_decode_prometheus_remote_write_destroy(context); if (result != 0) { - flb_plg_debug(ctx->ins, "could not ingest metrics : %d", result); - return -1; + flb_plg_error(ctx->ins, "could not ingest metrics : %d", result); + return 500; } return 0; @@ -115,7 +118,12 @@ int prom_rw_prot_handle_ng(struct flb_http_request *request, result = process_payload_metrics_ng(ctx, request); if (result != 0) { - send_response_ng(response, 400, "error: invalid request\n"); + if (result >= 500) { + send_response_ng(response, result, "error: could not ingest metrics\n"); + } + else { + send_response_ng(response, result, "error: invalid request\n"); + } return -1; } From 352c3ce969c3dd61c9ca12746154951a1ce64c25 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:06 -0600 Subject: [PATCH 16/42] in_splunk: return 400 on missing host header Signed-off-by: Eduardo Silva --- plugins/in_splunk/splunk_prot.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/in_splunk/splunk_prot.c b/plugins/in_splunk/splunk_prot.c index f2048fdbde8..efaf93f062b 100644 --- a/plugins/in_splunk/splunk_prot.c +++ b/plugins/in_splunk/splunk_prot.c @@ -576,8 +576,6 @@ static int validate_auth_header_ng(struct flb_splunk *ctx, struct flb_http_reque else { return SPLUNK_AUTH_MISSING_CRED; } - - return SPLUNK_AUTH_SUCCESS; } static int process_hec_payload_ng(struct flb_http_request *request, @@ -688,7 +686,7 @@ int splunk_prot_handle_ng(struct flb_http_request *request, /* HTTP/1.1 needs Host header */ if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { - + send_response_ng(response, 400, "error: missing host header\n"); return -1; } From 77f2bb93027802c4f0f89e66783303aebc76ae40 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:13 -0600 Subject: [PATCH 17/42] out_prometheus_exporter: return empty metrics payload before first snapshot Signed-off-by: Eduardo Silva --- plugins/out_prometheus_exporter/prom_http.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/out_prometheus_exporter/prom_http.c b/plugins/out_prometheus_exporter/prom_http.c index eed6cc76662..49a375640eb 100644 --- a/plugins/out_prometheus_exporter/prom_http.c +++ b/plugins/out_prometheus_exporter/prom_http.c @@ -27,6 +27,7 @@ static int prom_http_request_handler(struct flb_http_request *request, struct flb_http_response *response) { cfl_sds_t payload; + cfl_sds_t source_payload; struct prom_exporter *ctx; struct prom_http *ph; @@ -51,14 +52,25 @@ static int prom_http_request_handler(struct flb_http_request *request, } pthread_mutex_lock(&ph->metrics_mutex); - payload = ph->metrics_payload; - if (payload != NULL) { - payload = cfl_sds_create_len(payload, cfl_sds_len(payload)); + source_payload = ph->metrics_payload; + if (source_payload != NULL) { + payload = cfl_sds_create_len(source_payload, cfl_sds_len(source_payload)); + } + else { + payload = NULL; } pthread_mutex_unlock(&ph->metrics_mutex); if (payload == NULL) { - flb_http_response_set_status(response, 404); + if (source_payload == NULL) { + return flb_hs_response_set_payload(response, + 200, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + NULL, + 0); + } + + flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } flb_hs_response_set_payload(response, From 788135702f4f26c615a243f7a8695e6db621cc3d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:21 -0600 Subject: [PATCH 18/42] help: fix HTTP server option map handling Signed-off-by: Eduardo Silva --- src/flb_help.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/flb_help.c b/src/flb_help.c index eda74f92f00..8aedcad2e75 100644 --- a/src/flb_help.c +++ b/src/flb_help.c @@ -201,6 +201,7 @@ int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_s { struct mk_list *head; struct mk_list *config_map; + struct mk_list *http_config_map; struct flb_mp_map_header mh; struct flb_config_map *m; msgpack_sbuffer mp_sbuf; @@ -271,9 +272,9 @@ int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_s options_size += 3; } if ((ins->flags & FLB_INPUT_HTTP_SERVER) != 0) { - config_map = flb_http_server_get_config_map(ins->config); - options_size += mk_list_size(config_map); - flb_config_map_destroy(config_map); + http_config_map = flb_http_server_get_config_map(ins->config); + options_size += mk_list_size(http_config_map); + flb_config_map_destroy(http_config_map); } msgpack_pack_array(&mp_pck, options_size); @@ -284,12 +285,12 @@ int flb_help_input(struct flb_input_instance *ins, void **out_buf, size_t *out_s pack_config_map_entry(&mp_pck, &m_input_net_port); } if ((ins->flags & FLB_INPUT_HTTP_SERVER) != 0) { - config_map = flb_http_server_get_config_map(ins->config); - mk_list_foreach(head, config_map) { + http_config_map = flb_http_server_get_config_map(ins->config); + mk_list_foreach(head, http_config_map) { m = mk_list_entry(head, struct flb_config_map, _head); pack_config_map_entry(&mp_pck, m); } - flb_config_map_destroy(config_map); + flb_config_map_destroy(http_config_map); } mk_list_foreach(head, config_map) { @@ -485,12 +486,13 @@ int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out { struct mk_list *head; struct mk_list *config_map; + struct mk_list *http_config_map; struct flb_mp_map_header mh; struct flb_config_map *m; msgpack_sbuffer mp_sbuf; msgpack_packer mp_pck; int options_size = 0; - struct mk_list *tls_config; + struct mk_list *tls_config_map; struct flb_config_map m_output_net_host = { .type = FLB_CONFIG_MAP_STR, .name = "host", @@ -545,20 +547,18 @@ int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out pack_str(&mp_pck, "options"); config_map = flb_config_map_create(ins->config, ins->p->config_map); - options_size = mk_list_size(config_map); - options_size = mk_list_size(config_map); if (ins->flags & FLB_OUTPUT_NET) { options_size += 2; } if (ins->flags & FLB_OUTPUT_HTTP_SERVER) { - config_map = flb_http_server_get_config_map(ins->config); - options_size += mk_list_size(config_map); - flb_config_map_destroy(config_map); + http_config_map = flb_http_server_get_config_map(ins->config); + options_size += mk_list_size(http_config_map); + flb_config_map_destroy(http_config_map); } if (ins->flags & FLB_IO_OPT_TLS) { - tls_config = flb_tls_get_config_map(ins->config); - options_size += mk_list_size(tls_config); + tls_config_map = flb_tls_get_config_map(ins->config); + options_size += mk_list_size(tls_config_map); } msgpack_pack_array(&mp_pck, options_size); @@ -568,19 +568,19 @@ int flb_help_output(struct flb_output_instance *ins, void **out_buf, size_t *out pack_config_map_entry(&mp_pck, &m_output_net_port); } if (ins->flags & FLB_OUTPUT_HTTP_SERVER) { - config_map = flb_http_server_get_config_map(ins->config); - mk_list_foreach(head, config_map) { + http_config_map = flb_http_server_get_config_map(ins->config); + mk_list_foreach(head, http_config_map) { m = mk_list_entry(head, struct flb_config_map, _head); pack_config_map_entry(&mp_pck, m); } - flb_config_map_destroy(config_map); + flb_config_map_destroy(http_config_map); } if (ins->flags & FLB_IO_OPT_TLS) { - mk_list_foreach(head, tls_config) { + mk_list_foreach(head, tls_config_map) { m = mk_list_entry(head, struct flb_config_map, _head); pack_config_map_entry(&mp_pck, m); } - flb_config_map_destroy(tls_config); + flb_config_map_destroy(tls_config_map); } mk_list_foreach(head, config_map) { From 8ccb6037ae7fccef6d4c779dbb67785c6297bf9a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:28 -0600 Subject: [PATCH 19/42] input: free http server config on ring buffer init failure Signed-off-by: Eduardo Silva --- src/flb_input.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/flb_input.c b/src/flb_input.c index 2f4a13dec42..e88e45d914e 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -446,6 +446,9 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, if (!instance->rb) { flb_error("instance %s could not initialize ring buffer", flb_input_name(instance)); + if (instance->http_server_config != NULL) { + flb_free(instance->http_server_config); + } flb_free(instance); return NULL; } From a8b20e30247cf6ac1536da517e4be2105596d2ee Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:42 -0600 Subject: [PATCH 20/42] http_server: fix endpoint registration and response handling Signed-off-by: Eduardo Silva --- src/http_server/api/v1/health.c | 2 +- src/http_server/api/v1/metrics.c | 10 ++++++++-- src/http_server/api/v1/trace.c | 1 + src/http_server/api/v2/metrics.c | 18 ++++++++++++------ src/http_server/api/v2/reload.c | 4 ++-- src/http_server/flb_hs.c | 15 ++++++++++++++- src/http_server/flb_hs_utils.c | 4 +++- 7 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/http_server/api/v1/health.c b/src/http_server/api/v1/health.c index f3ee3abc2b9..55b295c7558 100644 --- a/src/http_server/api/v1/health.c +++ b/src/http_server/api/v1/health.c @@ -146,7 +146,6 @@ static int cb_health(struct flb_hs *hs, int ret; int status = is_healthy(hs); - (void) hs; (void) request; if (status == FLB_TRUE) { @@ -170,5 +169,6 @@ int api_v1_health(struct flb_hs *hs) void flb_hs_health_destroy() { + /* cleanup handled by flb_hs lifecycle */ return; } diff --git a/src/http_server/api/v1/metrics.c b/src/http_server/api/v1/metrics.c index 991c84a610a..2ad8e2e2309 100644 --- a/src/http_server/api/v1/metrics.c +++ b/src/http_server/api/v1/metrics.c @@ -428,8 +428,14 @@ static int cb_metrics(struct flb_hs *hs, /* Perform registration */ int api_v1_metrics(struct flb_hs *hs) { - flb_hs_register_endpoint(hs, "/api/v1/metrics/prometheus", - FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + int ret; + + ret = flb_hs_register_endpoint(hs, "/api/v1/metrics/prometheus", + FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + if (ret != 0) { + return ret; + } + return flb_hs_register_endpoint(hs, "/api/v1/metrics", FLB_HS_ROUTE_EXACT, cb_metrics); } diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index c82b8e6ae8f..33605651821 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -598,6 +598,7 @@ static int cb_traces(struct flb_hs *hs, } else if (request->method == HTTP_METHOD_DELETE) { disable_trace_input(hs, input_name, input_nlen); + msgpack_pack_map(&mp_pck, 1); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_OK, HTTP_RESULT_OK_LEN); } diff --git a/src/http_server/api/v2/metrics.c b/src/http_server/api/v2/metrics.c index cbe02af9a89..217bd3d6612 100644 --- a/src/http_server/api/v2/metrics.c +++ b/src/http_server/api/v2/metrics.c @@ -58,17 +58,17 @@ static int cb_metrics_prometheus(struct flb_hs *hs, return flb_http_response_commit(response); } + buf->users++; cmt = (struct cmt *) buf->raw_data; /* convert CMetrics to text */ payload = cmt_encode_prometheus_create(cmt, CMT_FALSE); if (!payload) { + buf->users--; flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } - buf->users++; - flb_hs_response_set_payload(response, 200, FLB_HS_CONTENT_TYPE_PROMETHEUS, payload, cfl_sds_len(payload)); @@ -96,17 +96,17 @@ static int cb_metrics(struct flb_hs *hs, return flb_http_response_commit(response); } + buf->users++; cmt = (struct cmt *) buf->raw_data; /* convert CMetrics to text */ payload = cmt_encode_text_create(cmt); if (!payload) { + buf->users--; flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } - buf->users++; - flb_hs_response_set_payload(response, 200, FLB_HS_CONTENT_TYPE_OTHER, payload, cfl_sds_len(payload)); @@ -120,8 +120,14 @@ static int cb_metrics(struct flb_hs *hs, /* Perform registration */ int api_v2_metrics(struct flb_hs *hs) { - flb_hs_register_endpoint(hs, "/api/v2/metrics/prometheus", - FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + int ret; + + ret = flb_hs_register_endpoint(hs, "/api/v2/metrics/prometheus", + FLB_HS_ROUTE_EXACT, cb_metrics_prometheus); + if (ret != 0) { + return ret; + } + return flb_hs_register_endpoint(hs, "/api/v2/metrics", FLB_HS_ROUTE_EXACT, cb_metrics); } diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index cd8953f35a4..d31879a9305 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -117,7 +117,7 @@ static int handle_reload_request(struct flb_http_response *response, out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { - flb_http_response_set_status(response, 400); + flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } out_size = flb_sds_len(out_buf); @@ -151,7 +151,7 @@ static int handle_get_reload_status(struct flb_http_response *response, out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { - flb_http_response_set_status(response, 400); + flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } out_size = flb_sds_len(out_buf); diff --git a/src/http_server/flb_hs.c b/src/http_server/flb_hs.c index 42a6995d407..cde868a7b16 100644 --- a/src/http_server/flb_hs.c +++ b/src/http_server/flb_hs.c @@ -26,6 +26,8 @@ #include #include +#include + /* v1 */ #include "api/v1/register.h" #include "api/v1/health.h" @@ -142,6 +144,7 @@ int flb_hs_register_endpoint(struct flb_hs *hs, return -1; } + /* Registered endpoints are static literals owned by the caller/module. */ route->path = path; route->match_type = match_type; route->callback = callback; @@ -285,6 +288,8 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, struct flb_config *config) { int ret; + char *end; + long port; struct flb_hs *hs; struct flb_http_server_options options; @@ -311,7 +316,15 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, options.request_callback = flb_hs_request_handler; options.user_data = hs; options.address = (char *) listen; - options.port = atoi(tcp_port); + errno = 0; + port = strtol(tcp_port, &end, 10); + if (errno == ERANGE || end == tcp_port || *end != '\0' || + port <= 0 || port > 65535) { + flb_error("[http_server] invalid monitoring tcp_port '%s'", tcp_port); + flb_free(hs); + return NULL; + } + options.port = (int) port; options.networking_flags = 0; flb_net_setup_init(&hs->net_setup); options.networking_setup = &hs->net_setup; diff --git a/src/http_server/flb_hs_utils.c b/src/http_server/flb_hs_utils.c index dc78ad3ab51..662212e7b47 100644 --- a/src/http_server/flb_hs_utils.c +++ b/src/http_server/flb_hs_utils.c @@ -86,7 +86,9 @@ int flb_hs_response_set_payload(struct flb_http_response *response, } flb_http_response_set_status(response, status); - flb_hs_response_set_content_type(response, type); + if (flb_hs_response_set_content_type(response, type) != 0) { + return -1; + } if (payload != NULL && payload_size > 0) { flb_http_response_set_body(response, (unsigned char *) payload, payload_size); From 2b62a256417b6285e8b41910cb3f217613deec22 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:22:48 -0600 Subject: [PATCH 21/42] tests: internal: destroy HTTP server test instance Signed-off-by: Eduardo Silva --- tests/internal/http_server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/internal/http_server.c b/tests/internal/http_server.c index 20d425c02ce..516340fcd20 100644 --- a/tests/internal/http_server.c +++ b/tests/internal/http_server.c @@ -178,6 +178,7 @@ void test_http_server_managed_worker_contract() TEST_CHECK(context.init_calls == 0); TEST_CHECK(context.exit_calls == 0); + flb_http_server_destroy(&server); test_http_server_context_destroy(&context); flb_config_exit(config); } From b93e63a161cc6a2e89e296b344633ef8b0b4c215 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 16:27:42 -0600 Subject: [PATCH 22/42] lib: monkey: upgrade to v1.8.7 Signed-off-by: Eduardo Silva --- lib/monkey/CMakeLists.txt | 2 +- lib/monkey/mk_server/mk_server.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/monkey/CMakeLists.txt b/lib/monkey/CMakeLists.txt index 9797f474e41..3dc1ef28a01 100644 --- a/lib/monkey/CMakeLists.txt +++ b/lib/monkey/CMakeLists.txt @@ -23,7 +23,7 @@ endif() # Monkey Version set(MK_VERSION_MAJOR 1) set(MK_VERSION_MINOR 8) -set(MK_VERSION_PATCH 6) +set(MK_VERSION_PATCH 7) set(MK_VERSION_STR "${MK_VERSION_MAJOR}.${MK_VERSION_MINOR}.${MK_VERSION_PATCH}") # Output paths diff --git a/lib/monkey/mk_server/mk_server.c b/lib/monkey/mk_server/mk_server.c index a84ef448571..82c7a403b95 100644 --- a/lib/monkey/mk_server/mk_server.c +++ b/lib/monkey/mk_server/mk_server.c @@ -578,6 +578,10 @@ void mk_server_worker_loop(struct mk_server *server) if (timeout_fd > 0) { mk_event_timeout_destroy(evl, server_timeout); } + + mk_sched_threads_destroy_all(sched); + mk_sched_event_free_all(sched); + mk_mem_free(MK_TLS_GET(mk_tls_server_timeout)); mk_server_listen_exit(sched->listeners); mk_event_loop_destroy(evl); From 623b463717313a8be40ac7b20f7994d90f6d1d8a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:11:09 -0600 Subject: [PATCH 23/42] in_prometheus_remote_write: delay context assignment until server startup Signed-off-by: Eduardo Silva --- plugins/in_prometheus_remote_write/prom_rw.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/in_prometheus_remote_write/prom_rw.c b/plugins/in_prometheus_remote_write/prom_rw.c index 69508734ac2..8b73a92eb06 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.c +++ b/plugins/in_prometheus_remote_write/prom_rw.c @@ -48,9 +48,6 @@ static int prom_rw_init(struct flb_input_instance *ins, return -1; } - /* Set the context */ - flb_input_set_context(ins, ctx); - ret = flb_input_http_server_options_init(&http_server_options, ins, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | @@ -79,6 +76,8 @@ static int prom_rw_init(struct flb_input_instance *ins, return -1; } + flb_input_set_context(ins, ctx); + flb_plg_info(ctx->ins, "listening on %s:%u with %i worker%s", ins->host.listen, From d28ce0bb30366a0a5c23258929f73ddcad5edce8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:11:16 -0600 Subject: [PATCH 24/42] in_http: accept JSON content types with optional whitespace Signed-off-by: Eduardo Silva --- plugins/in_http/http_prot.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/plugins/in_http/http_prot.c b/plugins/in_http/http_prot.c index e4b3ca70a9a..5aa349cf407 100644 --- a/plugins/in_http/http_prot.c +++ b/plugins/in_http/http_prot.c @@ -41,26 +41,30 @@ static int process_pack_ng(struct flb_http *ctx, flb_sds_t tag, static int content_type_is_json(const char *content_type) { - size_t length; + const char *separator; + size_t media_type_length; if (content_type == NULL) { return FLB_FALSE; } - length = strlen(content_type); + separator = strchr(content_type, ';'); + if (separator == NULL) { + return strcasecmp(content_type, "application/json") == 0; + } - if (length == 16 && - strncasecmp(content_type, "application/json", 16) == 0) { - return FLB_TRUE; + media_type_length = separator - content_type; + + while (media_type_length > 0 && + isspace((unsigned char) content_type[media_type_length - 1])) { + media_type_length--; } - if (length > 16 && - (strncasecmp(content_type, "application/json;", 17) == 0 || - strncasecmp(content_type, "application/json ", 17) == 0)) { - return FLB_TRUE; + if (media_type_length != strlen("application/json")) { + return FLB_FALSE; } - return FLB_FALSE; + return strncasecmp(content_type, "application/json", media_type_length) == 0; } static int content_type_is_urlencoded(const char *content_type) From b83aba2d6734a33d2c31f0e908a190f83487914d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:11:24 -0600 Subject: [PATCH 25/42] out_prometheus_exporter: propagate metrics response errors Signed-off-by: Eduardo Silva --- plugins/out_prometheus_exporter/prom_http.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/out_prometheus_exporter/prom_http.c b/plugins/out_prometheus_exporter/prom_http.c index 49a375640eb..64196d66533 100644 --- a/plugins/out_prometheus_exporter/prom_http.c +++ b/plugins/out_prometheus_exporter/prom_http.c @@ -26,6 +26,7 @@ static int prom_http_request_handler(struct flb_http_request *request, struct flb_http_response *response) { + int ret; cfl_sds_t payload; cfl_sds_t source_payload; struct prom_exporter *ctx; @@ -73,15 +74,15 @@ static int prom_http_request_handler(struct flb_http_request *request, flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } - flb_hs_response_set_payload(response, - 200, - FLB_HS_CONTENT_TYPE_PROMETHEUS, - payload, - cfl_sds_len(payload)); + ret = flb_hs_response_set_payload(response, + 200, + FLB_HS_CONTENT_TYPE_PROMETHEUS, + payload, + cfl_sds_len(payload)); cfl_sds_destroy(payload); - return 0; + return ret; } struct prom_http *prom_http_server_create(struct prom_exporter *ctx, From e9d318f3358e5cff5f5a3bbf20fc37a31a2f6ab5 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:12:59 -0600 Subject: [PATCH 26/42] input: fix early cleanup in HTTP server property and instance setup paths Signed-off-by: Eduardo Silva --- src/flb_input.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/flb_input.c b/src/flb_input.c index e88e45d914e..6f0bb3cf5eb 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -333,6 +333,10 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, ctx = flb_calloc(1, sizeof(struct flb_plugin_input_proxy_context)); if (!ctx) { flb_errno(); + flb_hash_table_destroy(instance->ht_log_chunks); + flb_hash_table_destroy(instance->ht_metric_chunks); + flb_hash_table_destroy(instance->ht_trace_chunks); + flb_hash_table_destroy(instance->ht_profile_chunks); flb_free(instance->http_server_config); flb_free(instance); return NULL; @@ -446,9 +450,18 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, if (!instance->rb) { flb_error("instance %s could not initialize ring buffer", flb_input_name(instance)); - if (instance->http_server_config != NULL) { - flb_free(instance->http_server_config); + flb_hash_table_destroy(instance->ht_log_chunks); + flb_hash_table_destroy(instance->ht_metric_chunks); + flb_hash_table_destroy(instance->ht_trace_chunks); + flb_hash_table_destroy(instance->ht_profile_chunks); + if (plugin->type != FLB_INPUT_PLUGIN_CORE) { + flb_free(instance->context); } + flb_sds_destroy(instance->host.name); + flb_sds_destroy(instance->host.address); + flb_uri_destroy(instance->host.uri); + flb_sds_destroy(instance->host.listen); + flb_free(instance->http_server_config); flb_free(instance); return NULL; } @@ -673,9 +686,7 @@ int flb_input_set_property(struct flb_input_instance *ins, tmp != NULL) { kv = flb_kv_item_create(&ins->http_server_properties, (char *) k, NULL); if (!kv) { - if (tmp) { - flb_sds_destroy(tmp); - } + flb_sds_destroy(tmp); return -1; } kv->val = tmp; From 4dac9f65e2203fa003429aa29ca92f560ba6502b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:13:06 -0600 Subject: [PATCH 27/42] http_server: fix trace and reload error handling Signed-off-by: Eduardo Silva --- src/http_server/api/v1/trace.c | 19 ++++++++++++++----- src/http_server/api/v2/reload.c | 2 ++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index 33605651821..2e0168f4b59 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -626,7 +626,7 @@ static int cb_traces(struct flb_hs *hs, else if (http_status == 503) { msgpack_pack_map(&mp_pck, 2); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_STATUS, HTTP_FIELD_STATUS_LEN); - msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_OK, HTTP_RESULT_OK_LEN); + msgpack_pack_str_with_body(&mp_pck, HTTP_RESULT_ERROR, HTTP_RESULT_ERROR_LEN); msgpack_pack_str_with_body(&mp_pck, HTTP_FIELD_MESSAGE, HTTP_FIELD_MESSAGE_LEN); if (error_msg) { msgpack_pack_str_with_body(&mp_pck, error_msg, flb_sds_len(error_msg)); @@ -661,11 +661,20 @@ static int cb_traces(struct flb_hs *hs, /* Perform registration */ int api_v1_trace(struct flb_hs *hs) { + int ret; + if (hs->config->enable_chunk_trace == FLB_TRUE) { - flb_hs_register_endpoint(hs, "/api/v1/traces/", - FLB_HS_ROUTE_EXACT, cb_traces); - flb_hs_register_endpoint(hs, "/api/v1/trace/", - FLB_HS_ROUTE_PREFIX, cb_trace); + ret = flb_hs_register_endpoint(hs, "/api/v1/traces/", + FLB_HS_ROUTE_EXACT, cb_traces); + if (ret != 0) { + return ret; + } + + ret = flb_hs_register_endpoint(hs, "/api/v1/trace/", + FLB_HS_ROUTE_PREFIX, cb_trace); + if (ret != 0) { + return ret; + } } return 0; } diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index d31879a9305..9847a422f49 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -71,6 +71,7 @@ static int handle_reload_request(struct flb_http_response *response, else { ret = GenerateConsoleCtrlEvent(1 /* CTRL_BREAK_EVENT_1 */, 0); if (ret == 0) { + msgpack_sbuffer_destroy(&mp_sbuf); flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } @@ -100,6 +101,7 @@ static int handle_reload_request(struct flb_http_response *response, else { ret = kill(getpid(), SIGHUP); if (ret != 0) { + msgpack_sbuffer_destroy(&mp_sbuf); flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } From bfc799ef44626766dd4a2cb7bd6636c1a0b9e7eb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:13:19 -0600 Subject: [PATCH 28/42] http_server: harden cached buffer lifetime and endpoint registration Signed-off-by: Eduardo Silva --- include/fluent-bit/http_server/flb_hs.h | 3 ++ src/http_server/api/v1/metrics.c | 13 +++--- src/http_server/api/v1/register.c | 34 +++++++++++--- src/http_server/api/v1/storage.c | 3 +- src/http_server/api/v2/metrics.c | 9 ++-- src/http_server/flb_hs.c | 59 +++++++++++++++++++++++-- 6 files changed, 101 insertions(+), 20 deletions(-) diff --git a/include/fluent-bit/http_server/flb_hs.h b/include/fluent-bit/http_server/flb_hs.h index f066f1195dc..3e25de019c8 100644 --- a/include/fluent-bit/http_server/flb_hs.h +++ b/include/fluent-bit/http_server/flb_hs.h @@ -33,6 +33,7 @@ */ struct flb_hs_buf { int users; + int pending_free; flb_sds_t data; void *raw_data; size_t raw_size; @@ -100,6 +101,8 @@ int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size); int flb_hs_destroy(struct flb_hs *ctx); int flb_hs_start(struct flb_hs *hs); +void flb_hs_cmt_buffer_destroy(void *data); +void flb_hs_buf_release(struct flb_hs_buf *buffer, void (*raw_free)(void *)); int flb_hs_register_endpoint(struct flb_hs *hs, const char *path, int match_type, diff --git a/src/http_server/api/v1/metrics.c b/src/http_server/api/v1/metrics.c index 2ad8e2e2309..af5dbad2680 100644 --- a/src/http_server/api/v1/metrics.c +++ b/src/http_server/api/v1/metrics.c @@ -28,6 +28,7 @@ #include "metrics.h" #include +#include #include #include @@ -166,7 +167,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, if (!sds) { flb_http_response_set_status(response, 500); flb_http_response_commit(response); - buf->users--; + flb_hs_buf_release(buf, NULL); return 0; } @@ -176,7 +177,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, flb_sds_destroy(sds); flb_http_response_set_status(response, 500); flb_http_response_commit(response); - buf->users--; + flb_hs_buf_release(buf, NULL); return 0; } metric_helptxt_head = FLB_SDS_HEADER(metric_helptxt); @@ -205,7 +206,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, if (!metrics_arr) { flb_errno(); - buf->users--; + flb_hs_buf_release(buf, NULL); flb_http_response_set_status(response, 500); flb_http_response_commit(response); @@ -371,7 +372,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, #endif msgpack_unpacked_destroy(&result); - buf->users--; + flb_hs_buf_release(buf, NULL); flb_hs_response_set_payload(response, 200, FLB_HS_CONTENT_TYPE_PROMETHEUS, @@ -388,7 +389,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, error: flb_http_response_set_status(response, 500); flb_http_response_commit(response); - buf->users--; + flb_hs_buf_release(buf, NULL); for (i = 0; i < index; i++) { flb_sds_destroy(metrics_arr[i]); @@ -421,7 +422,7 @@ static int cb_metrics(struct flb_hs *hs, FLB_HS_CONTENT_TYPE_JSON, buf->data, flb_sds_len(buf->data)); - buf->users--; + flb_hs_buf_release(buf, NULL); return 0; } diff --git a/src/http_server/api/v1/register.c b/src/http_server/api/v1/register.c index 0c4117a2467..f2985821e72 100644 --- a/src/http_server/api/v1/register.c +++ b/src/http_server/api/v1/register.c @@ -29,20 +29,42 @@ int api_v1_registration(struct flb_hs *hs) { - api_v1_uptime(hs); - api_v1_metrics(hs); - api_v1_plugins(hs); + int ret; + + ret = api_v1_uptime(hs); + if (ret != 0) { + return ret; + } + + ret = api_v1_metrics(hs); + if (ret != 0) { + return ret; + } + + ret = api_v1_plugins(hs); + if (ret != 0) { + return ret; + } #ifdef FLB_HAVE_CHUNK_TRACE - api_v1_trace(hs); + ret = api_v1_trace(hs); + if (ret != 0) { + return ret; + } #endif /* FLB_HAVE_CHUNK_TRACE */ if (hs->config->health_check == FLB_TRUE) { - api_v1_health(hs); + ret = api_v1_health(hs); + if (ret != 0) { + return ret; + } } if (hs->config->storage_metrics == FLB_TRUE) { - api_v1_storage_metrics(hs); + ret = api_v1_storage_metrics(hs); + if (ret != 0) { + return ret; + } } return 0; diff --git a/src/http_server/api/v1/storage.c b/src/http_server/api/v1/storage.c index 77f51e842d0..4e5dd2365a7 100644 --- a/src/http_server/api/v1/storage.c +++ b/src/http_server/api/v1/storage.c @@ -23,6 +23,7 @@ #include "storage.h" #include +#include #include #include @@ -56,7 +57,7 @@ static int cb_storage(struct flb_hs *hs, FLB_HS_CONTENT_TYPE_JSON, buf->data, flb_sds_len(buf->data)); - buf->users--; + flb_hs_buf_release(buf, NULL); return 0; } diff --git a/src/http_server/api/v2/metrics.c b/src/http_server/api/v2/metrics.c index 217bd3d6612..f27cf48d826 100644 --- a/src/http_server/api/v2/metrics.c +++ b/src/http_server/api/v2/metrics.c @@ -28,6 +28,7 @@ #include "metrics.h" #include +#include #include #define null_check(x) do { if (!x) { goto error; } else {sds = x;} } while (0) @@ -64,7 +65,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, /* convert CMetrics to text */ payload = cmt_encode_prometheus_create(cmt, CMT_FALSE); if (!payload) { - buf->users--; + flb_hs_buf_release(buf, flb_hs_cmt_buffer_destroy); flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } @@ -75,7 +76,7 @@ static int cb_metrics_prometheus(struct flb_hs *hs, cmt_encode_prometheus_destroy(payload); - buf->users--; + flb_hs_buf_release(buf, flb_hs_cmt_buffer_destroy); return 0; } @@ -102,7 +103,7 @@ static int cb_metrics(struct flb_hs *hs, /* convert CMetrics to text */ payload = cmt_encode_text_create(cmt); if (!payload) { - buf->users--; + flb_hs_buf_release(buf, flb_hs_cmt_buffer_destroy); flb_http_response_set_status(response, 500); return flb_http_response_commit(response); } @@ -113,7 +114,7 @@ static int cb_metrics(struct flb_hs *hs, cmt_encode_text_destroy(payload); - buf->users--; + flb_hs_buf_release(buf, flb_hs_cmt_buffer_destroy); return 0; } diff --git a/src/http_server/flb_hs.c b/src/http_server/flb_hs.c index cde868a7b16..2d46787d5f6 100644 --- a/src/http_server/flb_hs.c +++ b/src/http_server/flb_hs.c @@ -67,6 +67,11 @@ static void flb_hs_destroy_cmt_buffer(void *data) cmt_destroy((struct cmt *) data); } +void flb_hs_cmt_buffer_destroy(void *data) +{ + flb_hs_destroy_cmt_buffer(data); +} + static int cb_root(struct flb_hs *hs, struct flb_http_request *request, struct flb_http_response *response) @@ -112,6 +117,11 @@ static void flb_hs_buf_cleanup(struct flb_hs_buf *buffer, return; } + if (buffer->users > 0) { + buffer->pending_free = FLB_TRUE; + return; + } + if (buffer->data != NULL) { flb_sds_destroy(buffer->data); buffer->data = NULL; @@ -128,9 +138,23 @@ static void flb_hs_buf_cleanup(struct flb_hs_buf *buffer, } buffer->raw_size = 0; + buffer->pending_free = FLB_FALSE; buffer->users = 0; } +void flb_hs_buf_release(struct flb_hs_buf *buffer, void (*raw_free)(void *)) +{ + if (buffer == NULL || buffer->users <= 0) { + return; + } + + buffer->users--; + + if (buffer->users == 0 && buffer->pending_free == FLB_TRUE) { + flb_hs_buf_cleanup(buffer, raw_free); + } +} + int flb_hs_register_endpoint(struct flb_hs *hs, const char *path, int match_type, @@ -172,6 +196,9 @@ int flb_hs_push_health_metrics(struct flb_hs *hs, void *data, size_t size) while (hs->health_counter.period_counter > hs->health_counter.period_limit && mk_list_size(&hs->health_metrics) > 0) { buf = mk_list_entry_first(&hs->health_metrics, struct flb_hs_hc_buf, _head); + if (buf->users > 0) { + break; + } mk_list_del(&buf->_head); flb_free(buf); hs->health_counter.period_counter--; @@ -219,6 +246,11 @@ int flb_hs_push_pipeline_metrics(struct flb_hs *hs, void *data, size_t size) memcpy(raw_buffer, data, size); flb_hs_buf_cleanup(&hs->metrics, NULL); + if (hs->metrics.pending_free == FLB_TRUE) { + flb_sds_destroy(json_buffer); + flb_free(raw_buffer); + return -1; + } hs->metrics.data = json_buffer; hs->metrics.raw_data = raw_buffer; @@ -244,6 +276,10 @@ int flb_hs_push_metrics(struct flb_hs *hs, void *data, size_t size) } flb_hs_buf_cleanup(&hs->metrics_v2, flb_hs_destroy_cmt_buffer); + if (hs->metrics_v2.pending_free == FLB_TRUE) { + flb_hs_destroy_cmt_buffer(cmt); + return -1; + } hs->metrics_v2.raw_data = cmt; @@ -275,6 +311,11 @@ int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size) memcpy(raw_buffer, data, size); flb_hs_buf_cleanup(&hs->storage_metrics, NULL); + if (hs->storage_metrics.pending_free == FLB_TRUE) { + flb_sds_destroy(json_buffer); + flb_free(raw_buffer); + return -1; + } hs->storage_metrics.data = json_buffer; hs->storage_metrics.raw_data = raw_buffer; @@ -339,13 +380,25 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, } /* Register endpoints for /api/v1 */ - api_v1_registration(hs); + ret = api_v1_registration(hs); + if (ret != 0) { + flb_hs_destroy(hs); + return NULL; + } /* Register endpoints for /api/v2 */ - api_v2_registration(hs); + ret = api_v2_registration(hs); + if (ret != 0) { + flb_hs_destroy(hs); + return NULL; + } /* Root */ - flb_hs_register_endpoint(hs, "/", FLB_HS_ROUTE_EXACT, cb_root); + ret = flb_hs_register_endpoint(hs, "/", FLB_HS_ROUTE_EXACT, cb_root); + if (ret != 0) { + flb_hs_destroy(hs); + return NULL; + } return hs; } From e2f4d14cc0daa8a3d9b5bff859c4d2bff4920cc7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:13:25 -0600 Subject: [PATCH 29/42] tests: internal: stop on HTTP server init failure Signed-off-by: Eduardo Silva --- tests/internal/http_server.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/internal/http_server.c b/tests/internal/http_server.c index 516340fcd20..e19387053e7 100644 --- a/tests/internal/http_server.c +++ b/tests/internal/http_server.c @@ -107,6 +107,7 @@ void test_http_server_options_multi_worker_magic() struct flb_net_setup net_setup; struct flb_http_server server; struct flb_http_server_options options; + int ret; config = flb_config_init(); if (!TEST_CHECK(config != NULL)) { @@ -126,7 +127,12 @@ void test_http_server_options_multi_worker_magic() options.workers = 2; options.max_connections = 7; - TEST_CHECK(flb_http_server_init_with_options(&server, &options) == 0); + ret = flb_http_server_init_with_options(&server, &options); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_config_exit(config); + return; + } TEST_CHECK(server.workers == 2); TEST_CHECK(server.reuse_port == FLB_TRUE); TEST_CHECK(server.use_caller_event_loop == FLB_FALSE); @@ -144,6 +150,7 @@ void test_http_server_managed_worker_contract() struct flb_http_server server; struct flb_http_server_options options; struct test_http_server_context context; + int ret; config = flb_config_init(); if (!TEST_CHECK(config != NULL)) { return; @@ -167,7 +174,13 @@ void test_http_server_managed_worker_contract() options.cb_worker_init = test_http_server_worker_init; options.cb_worker_exit = test_http_server_worker_exit; - TEST_CHECK(flb_http_server_init_with_options(&server, &options) == 0); + ret = flb_http_server_init_with_options(&server, &options); + TEST_CHECK(ret == 0); + if (ret != 0) { + test_http_server_context_destroy(&context); + flb_config_exit(config); + return; + } TEST_CHECK(server.workers == 2); TEST_CHECK(server.use_caller_event_loop == FLB_FALSE); TEST_CHECK(server.reuse_port == FLB_TRUE); From e13d4beb408d819ab9b0a3211dc04a6be5a2d461 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:25:10 -0600 Subject: [PATCH 30/42] http_server: add JSON health endpoint to API v2 Signed-off-by: Eduardo Silva --- src/http_server/api/v1/health.c | 43 +++++++---- src/http_server/api/v1/health.h | 10 +++ src/http_server/api/v2/CMakeLists.txt | 1 + src/http_server/api/v2/health.c | 106 ++++++++++++++++++++++++++ src/http_server/api/v2/health.h | 28 +++++++ src/http_server/api/v2/register.c | 20 ++++- 6 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 src/http_server/api/v2/health.c create mode 100644 src/http_server/api/v2/health.h diff --git a/src/http_server/api/v1/health.c b/src/http_server/api/v1/health.c index 55b295c7558..17ffce5e215 100644 --- a/src/http_server/api/v1/health.c +++ b/src/http_server/api/v1/health.c @@ -39,14 +39,22 @@ * the error number of the newest metrics message minus * the error number in oldest metrics of period */ -static int is_healthy(struct flb_hs *hs) +int flb_hs_health_state_get(struct flb_hs *hs, struct flb_hs_health_state *state) { struct flb_hs_hc_buf *buf; - int period_errors; - int period_retry_failure; + + if (hs == NULL || state == NULL) { + return -1; + } + + memset(state, 0, sizeof(struct flb_hs_health_state)); + state->error_limit = hs->health_counter.error_limit; + state->retry_failure_limit = hs->health_counter.retry_failure_limit; + state->period_limit = hs->health_counter.period_limit; if (mk_list_is_empty(&hs->health_metrics) == 0) { - return FLB_TRUE; + state->healthy = FLB_TRUE; + return 0; } /* Get the error metrics entry from the start time of current period */ @@ -63,18 +71,20 @@ static int is_healthy(struct flb_hs *hs) * the error count in current period = (current error count in total) - * (begin error count in the period) */ - period_errors = hs->health_counter.error_counter - buf->error_count; - period_retry_failure = hs->health_counter.retry_failure_counter - - buf->retry_failure_count; + state->errors = hs->health_counter.error_counter - buf->error_count; + state->retries_failed = hs->health_counter.retry_failure_counter - + buf->retry_failure_count; buf->users--; - if (period_errors > hs->health_counter.error_limit || - period_retry_failure > hs->health_counter.retry_failure_limit) { - - return FLB_FALSE; + if (state->errors > hs->health_counter.error_limit || + state->retries_failed > hs->health_counter.retry_failure_limit) { + state->healthy = FLB_FALSE; + return 0; } - return FLB_TRUE; + state->healthy = FLB_TRUE; + + return 0; } /* read the metrics from message queue and update the counter*/ @@ -144,11 +154,16 @@ static int cb_health(struct flb_hs *hs, struct flb_http_response *response) { int ret; - int status = is_healthy(hs); + struct flb_hs_health_state state; (void) request; - if (status == FLB_TRUE) { + if (flb_hs_health_state_get(hs, &state) != 0) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } + + if (state.healthy == FLB_TRUE) { ret = flb_hs_response_send_string(response, 200, FLB_HS_CONTENT_TYPE_OTHER, "ok\n"); } diff --git a/src/http_server/api/v1/health.h b/src/http_server/api/v1/health.h index a0a4415831b..33ea562ae2a 100644 --- a/src/http_server/api/v1/health.h +++ b/src/http_server/api/v1/health.h @@ -25,8 +25,18 @@ #include #include +struct flb_hs_health_state { + int healthy; + int errors; + int retries_failed; + int error_limit; + int retry_failure_limit; + int period_limit; +}; + /* health endpoint*/ int api_v1_health(struct flb_hs *hs); +int flb_hs_health_state_get(struct flb_hs *hs, struct flb_hs_health_state *state); void read_metrics(void *data, size_t size, int *error_count, int *retry_failure_count); diff --git a/src/http_server/api/v2/CMakeLists.txt b/src/http_server/api/v2/CMakeLists.txt index a9d590fbd7f..6b8dc283a0c 100644 --- a/src/http_server/api/v2/CMakeLists.txt +++ b/src/http_server/api/v2/CMakeLists.txt @@ -1,5 +1,6 @@ # api/v2 set(src + health.c metrics.c reload.c register.c diff --git a/src/http_server/api/v2/health.c b/src/http_server/api/v2/health.c new file mode 100644 index 00000000000..966772e07b4 --- /dev/null +++ b/src/http_server/api/v2/health.c @@ -0,0 +1,106 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "../v1/health.h" +#include + +#include "health.h" + +static int cb_health(struct flb_hs *hs, + struct flb_http_request *request, + struct flb_http_response *response) +{ + int status_code; + size_t out_size; + flb_sds_t out_buf; + msgpack_packer mp_pck; + msgpack_sbuffer mp_sbuf; + struct flb_hs_health_state state; + + (void) request; + + if (flb_hs_health_state_get(hs, &state) != 0) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&mp_pck, 6); + msgpack_pack_str(&mp_pck, 6); + msgpack_pack_str_body(&mp_pck, "status", 6); + + if (state.healthy == FLB_TRUE) { + msgpack_pack_str(&mp_pck, 2); + msgpack_pack_str_body(&mp_pck, "ok", 2); + status_code = 200; + } + else { + msgpack_pack_str(&mp_pck, 5); + msgpack_pack_str_body(&mp_pck, "error", 5); + status_code = 500; + } + + msgpack_pack_str(&mp_pck, 6); + msgpack_pack_str_body(&mp_pck, "errors", 6); + msgpack_pack_int64(&mp_pck, state.errors); + + msgpack_pack_str(&mp_pck, 15); + msgpack_pack_str_body(&mp_pck, "retries_failed", 15); + msgpack_pack_int64(&mp_pck, state.retries_failed); + + msgpack_pack_str(&mp_pck, 11); + msgpack_pack_str_body(&mp_pck, "error_limit", 11); + msgpack_pack_int64(&mp_pck, state.error_limit); + + msgpack_pack_str(&mp_pck, 19); + msgpack_pack_str_body(&mp_pck, "retry_failure_limit", 19); + msgpack_pack_int64(&mp_pck, state.retry_failure_limit); + + msgpack_pack_str(&mp_pck, 12); + msgpack_pack_str_body(&mp_pck, "period_limit", 12); + msgpack_pack_int64(&mp_pck, state.period_limit); + + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); + msgpack_sbuffer_destroy(&mp_sbuf); + if (out_buf == NULL) { + flb_http_response_set_status(response, 500); + return flb_http_response_commit(response); + } + + out_size = flb_sds_len(out_buf); + flb_hs_response_set_payload(response, status_code, + FLB_HS_CONTENT_TYPE_JSON, + out_buf, out_size); + flb_sds_destroy(out_buf); + + return 0; +} + +int api_v2_health(struct flb_hs *hs) +{ + return flb_hs_register_endpoint(hs, "/api/v2/health", + FLB_HS_ROUTE_EXACT, cb_health); +} diff --git a/src/http_server/api/v2/health.h b/src/http_server/api/v2/health.h new file mode 100644 index 00000000000..de92c621c5a --- /dev/null +++ b/src/http_server/api/v2/health.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HS_API_V2_HEALTH_H +#define FLB_HS_API_V2_HEALTH_H + +#include +#include + +int api_v2_health(struct flb_hs *hs); + +#endif diff --git a/src/http_server/api/v2/register.c b/src/http_server/api/v2/register.c index 4c0a4e0ae8f..a57ac41480c 100644 --- a/src/http_server/api/v2/register.c +++ b/src/http_server/api/v2/register.c @@ -20,12 +20,28 @@ #include #include +#include "health.h" #include "metrics.h" #include "reload.h" int api_v2_registration(struct flb_hs *hs) { - api_v2_reload(hs); - api_v2_metrics(hs); + int ret; + + ret = api_v2_reload(hs); + if (ret != 0) { + return ret; + } + + ret = api_v2_metrics(hs); + if (ret != 0) { + return ret; + } + + ret = api_v2_health(hs); + if (ret != 0) { + return ret; + } + return 0; } From a5183b8f0521cadf2fe5d4be7c7288e81b524295 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:25:30 -0600 Subject: [PATCH 31/42] github: linter: allow http_server umbrella prefix for HTTP server paths Signed-off-by: Eduardo Silva --- .github/scripts/commit_prefix_check.py | 23 ++++++++++++++++++++- .github/scripts/tests/test_commit_lint.py | 25 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/scripts/commit_prefix_check.py b/.github/scripts/commit_prefix_check.py index 753e7bf4aa6..5c8e000d55f 100644 --- a/.github/scripts/commit_prefix_check.py +++ b/.github/scripts/commit_prefix_check.py @@ -127,6 +127,18 @@ def infer_prefix_from_paths(paths): return prefixes, build_optional +def is_http_server_interface_path(path): + p = path.replace(os.sep, "/") + + return ( + p.startswith("src/http_server/") + or p == "src/flb_http_common.c" + or p.startswith("include/fluent-bit/http_server/") + or p == "include/fluent-bit/flb_http_common.h" + or p == "HTTP_SERVER_API.md" + ) + + # ------------------------------------------------ # detect_bad_squash() must satisfy the tests EXACTLY # ------------------------------------------------ @@ -243,7 +255,7 @@ def validate_commit(commit): } # Prefixes that are allowed to cover multiple subcomponents - umbrella_prefixes = {"lib:", "tests:"} + umbrella_prefixes = {"lib:", "tests:", "http_server:"} # If more than one non-build prefix is inferred AND the subject is not an umbrella # prefix, check if the subject prefix is in the expected list. If it is, allow it @@ -271,6 +283,15 @@ def validate_commit(commit): f"Expected one of: {expected_str}" ) + elif subj_lower == "http_server:": + if not all(is_http_server_interface_path(p) for p in norm_paths): + expected_list = sorted(expected) + expected_str = ", ".join(expected_list) + return False, ( + f"Subject prefix '{subject_prefix}' does not match files changed.\n" + f"Expected one of: {expected_str}" + ) + else: expected_list = sorted(expected) expected_str = ", ".join(expected_list) diff --git a/.github/scripts/tests/test_commit_lint.py b/.github/scripts/tests/test_commit_lint.py index 5773a202622..51e239d9d99 100644 --- a/.github/scripts/tests/test_commit_lint.py +++ b/.github/scripts/tests/test_commit_lint.py @@ -311,6 +311,31 @@ def test_error_multiple_prefixes_inferred_from_files(): assert "does not match files changed" in msg +def test_valid_http_server_umbrella_prefix(): + """ + Commits touching only the HTTP server interface boundary may use http_server:. + """ + commit = make_commit( + "http_server: unify listener startup\n\nSigned-off-by: User", + ["src/http_server/flb_hs.c", "src/flb_http_common.c"] + ) + ok, _ = validate_commit(commit) + assert ok is True + + +def test_error_http_server_umbrella_with_unrelated_component(): + """ + http_server: must not cover unrelated core files outside the HTTP interface. + """ + commit = make_commit( + "http_server: adjust startup\n\nSigned-off-by: User", + ["src/http_server/flb_hs.c", "src/flb_input.c"] + ) + ok, msg = validate_commit(commit) + assert ok is False + assert "does not match files changed" in msg + + # ----------------------------------------------------------- # Edge Cases From 6dbf820f0e57b0a297b47e4bfa7ac00839dac283 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:33:34 -0600 Subject: [PATCH 32/42] tests: internal: chunk_router: initialize HTTP server properties Signed-off-by: Eduardo Silva --- tests/internal/input_chunk_routes.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/internal/input_chunk_routes.c b/tests/internal/input_chunk_routes.c index 7c26ba1e808..36866033df8 100644 --- a/tests/internal/input_chunk_routes.c +++ b/tests/internal/input_chunk_routes.c @@ -115,6 +115,7 @@ static int init_test_config(struct flb_config *config, /* Initialize properties list (required by flb_input_instance_init) */ mk_list_init(&in->properties); mk_list_init(&in->net_properties); + mk_list_init(&in->http_server_properties); mk_list_init(&in->oauth2_jwt_properties); /* Initialize hash tables for chunks (required by flb_input_chunk_destroy) */ @@ -241,6 +242,7 @@ static void cleanup_test_routing_scenario(struct flb_input_chunk *ic, /* Release properties */ flb_kv_release(&in->properties); flb_kv_release(&in->net_properties); + flb_kv_release(&in->http_server_properties); flb_kv_release(&in->oauth2_jwt_properties); /* Destroy metrics (created by flb_input_instance_init) */ From a66434bff5eea996f174b413ed2ae6b6095375fa Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:35:42 -0600 Subject: [PATCH 33/42] input: clean up resources on host setup failure Signed-off-by: Eduardo Silva --- src/flb_input.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/flb_input.c b/src/flb_input.c index 6f0bb3cf5eb..e390383eb5b 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -392,6 +392,21 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, if (plugin->flags & (FLB_INPUT_NET | FLB_INPUT_NET_SERVER)) { ret = flb_net_host_set(plugin->name, &instance->host, input); if (ret != 0) { + if (instance->ht_log_chunks) { + flb_hash_table_destroy(instance->ht_log_chunks); + } + if (instance->ht_metric_chunks) { + flb_hash_table_destroy(instance->ht_metric_chunks); + } + if (instance->ht_trace_chunks) { + flb_hash_table_destroy(instance->ht_trace_chunks); + } + if (instance->ht_profile_chunks) { + flb_hash_table_destroy(instance->ht_profile_chunks); + } + if (plugin->type != FLB_INPUT_PLUGIN_CORE && instance->context) { + flb_free(instance->context); + } flb_free(instance->http_server_config); flb_free(instance); return NULL; From cee1c4ef37d6915ef31ccef3650ece0ef7722602 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:35:48 -0600 Subject: [PATCH 34/42] http_server: validate trace request bodies before parsing Signed-off-by: Eduardo Silva --- src/http_server/api/v1/trace.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index 2e0168f4b59..af655e34ca4 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -279,6 +279,11 @@ static int http_enable_trace(struct flb_http_request *request, void *data, } msgpack_unpacked_init(&result); + if (request->body == NULL) { + ret = 400; + flb_error("missing json parameters"); + goto unpack_error; + } rc = flb_pack_json(request->body, cfl_sds_len(request->body), &buf, &buf_size, &root_type, NULL); if (rc == -1) { @@ -514,6 +519,11 @@ static int cb_traces(struct flb_hs *hs, msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); msgpack_unpacked_init(&result); + if (request->body == NULL) { + http_status = 400; + error_msg = flb_sds_create("missing request body"); + goto unpack_error; + } ret = flb_pack_json(request->body, cfl_sds_len(request->body), &buf, &buf_size, &root_type, NULL); if (ret == -1) { From 3634e34ac105b1ae8951bcbe730fd8342ff12dbe Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:35:53 -0600 Subject: [PATCH 35/42] http_server: free endpoint state on monitoring init failure Signed-off-by: Eduardo Silva --- src/http_server/flb_hs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/http_server/flb_hs.c b/src/http_server/flb_hs.c index 2d46787d5f6..f49d0fcb2e2 100644 --- a/src/http_server/flb_hs.c +++ b/src/http_server/flb_hs.c @@ -362,6 +362,7 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, if (errno == ERANGE || end == tcp_port || *end != '\0' || port <= 0 || port > 65535) { flb_error("[http_server] invalid monitoring tcp_port '%s'", tcp_port); + flb_hs_endpoints_free(hs); flb_free(hs); return NULL; } @@ -375,6 +376,7 @@ struct flb_hs *flb_hs_create(const char *listen, const char *tcp_port, ret = flb_http_server_init_with_options(&hs->server, &options); if (ret != 0) { + flb_hs_endpoints_free(hs); flb_free(hs); return NULL; } From ae4736ba4c4c6f5c775ca0a6a461788a9297b624 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:36:00 -0600 Subject: [PATCH 36/42] http_server: return 405 for unsupported reload methods Signed-off-by: Eduardo Silva --- src/http_server/api/v2/reload.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index 9847a422f49..e0dd4004f56 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -180,7 +180,8 @@ static int cb_reload(struct flb_hs *hs, return handle_get_reload_status(response, config); } - flb_http_response_set_status(response, 400); + flb_http_response_set_status(response, 405); + flb_http_response_set_header(response, "Allow", 5, "GET, POST, PUT", 14); return flb_http_response_commit(response); } From ee53f6269fd5f37df7a01a5544b1ac0346d94d65 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:51:52 -0600 Subject: [PATCH 37/42] http_server: use pthread portability wrapper Signed-off-by: Eduardo Silva --- src/http_server/flb_http_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 6059487e57c..4eb3b48ac9a 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -21,8 +21,8 @@ #include #include #include +#include #include -#include #include #include From 18df4e8f86d1ad8f80eb98ddae21f047295bc8f4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 17:55:10 -0600 Subject: [PATCH 38/42] http_server: fix retries_failed field length in v2 health Signed-off-by: Eduardo Silva --- src/http_server/api/v2/health.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http_server/api/v2/health.c b/src/http_server/api/v2/health.c index 966772e07b4..9e4873cb9a2 100644 --- a/src/http_server/api/v2/health.c +++ b/src/http_server/api/v2/health.c @@ -67,8 +67,8 @@ static int cb_health(struct flb_hs *hs, msgpack_pack_str_body(&mp_pck, "errors", 6); msgpack_pack_int64(&mp_pck, state.errors); - msgpack_pack_str(&mp_pck, 15); - msgpack_pack_str_body(&mp_pck, "retries_failed", 15); + msgpack_pack_str(&mp_pck, 14); + msgpack_pack_str_body(&mp_pck, "retries_failed", 14); msgpack_pack_int64(&mp_pck, state.retries_failed); msgpack_pack_str(&mp_pck, 11); From 8b7bc23d0ff118872ab30bee6d1762b58fae15d8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 18:15:48 -0600 Subject: [PATCH 39/42] http_server: remove GCC-only unused function attribute Signed-off-by: Eduardo Silva --- src/http_server/flb_http_server.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 4eb3b48ac9a..848a94167ee 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -551,8 +551,7 @@ static void *flb_http_server_worker_thread(void *data) return NULL; } -static int __attribute__((unused)) -flb_http_server_runtime_start(struct flb_http_server *session) +static int flb_http_server_runtime_start(struct flb_http_server *session) { int index; int result; From bf537f3ef091fb2b3190fb0fcb9db00fa8465feb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 18:38:05 -0600 Subject: [PATCH 40/42] out_prometheus_exporter: use pthread portability wrapper Signed-off-by: Eduardo Silva --- plugins/out_prometheus_exporter/prom_http.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_prometheus_exporter/prom_http.h b/plugins/out_prometheus_exporter/prom_http.h index 06cd7b132f9..3c678485c31 100644 --- a/plugins/out_prometheus_exporter/prom_http.h +++ b/plugins/out_prometheus_exporter/prom_http.h @@ -21,8 +21,8 @@ #define FLB_PROMETHEUS_EXPORTER_HTTP_H #include +#include #include -#include #include "prom.h" From c919a659e68e752dd6f9f40af6cf18e214b23f50 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 18:38:10 -0600 Subject: [PATCH 41/42] out_vivo_exporter: use pthread portability wrapper Signed-off-by: Eduardo Silva --- plugins/out_vivo_exporter/vivo_stream.h | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/out_vivo_exporter/vivo_stream.h b/plugins/out_vivo_exporter/vivo_stream.h index 18d29b7d7ef..57a4b2aeb93 100644 --- a/plugins/out_vivo_exporter/vivo_stream.h +++ b/plugins/out_vivo_exporter/vivo_stream.h @@ -21,6 +21,7 @@ #define FLB_VIVO_STREAM_H #include +#include #include "vivo.h" From 877e961b43d7b248934fe077243e81ee728d32ec Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 11 Mar 2026 20:39:40 -0600 Subject: [PATCH 42/42] tests: internal: use pthread portability wrapper in http server test Signed-off-by: Eduardo Silva --- tests/internal/http_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/http_server.c b/tests/internal/http_server.c index e19387053e7..133d45a8e12 100644 --- a/tests/internal/http_server.c +++ b/tests/internal/http_server.c @@ -4,10 +4,10 @@ #include #include #include +#include #include #include -#include #include #include "flb_tests_internal.h"