http_server: unify Fluent Bit HTTP listeners on flb_http_server

.github/scripts/commit_prefix_check.py

@@ -127,6 +127,18 @@ def infer_prefix_from_paths(paths):
     return prefixes, build_optional
 
 
+def is_http_server_interface_path(path):
+    p = path.replace(os.sep, "/")
+
+    return (
+        p.startswith("src/http_server/")
+        or p == "src/flb_http_common.c"
+        or p.startswith("include/fluent-bit/http_server/")
+        or p == "include/fluent-bit/flb_http_common.h"
+        or p == "HTTP_SERVER_API.md"
+    )
+
+
 # ------------------------------------------------
 # detect_bad_squash() must satisfy the tests EXACTLY
 # ------------------------------------------------
@@ -243,7 +255,7 @@ def validate_commit(commit):
     }
 
     # Prefixes that are allowed to cover multiple subcomponents
-    umbrella_prefixes = {"lib:", "tests:"}
+    umbrella_prefixes = {"lib:", "tests:", "http_server:"}
 
     # If more than one non-build prefix is inferred AND the subject is not an umbrella
     # prefix, check if the subject prefix is in the expected list. If it is, allow it
@@ -271,6 +283,15 @@ def validate_commit(commit):
                         f"Expected one of: {expected_str}"
                     )
 
+            elif subj_lower == "http_server:":
+                if not all(is_http_server_interface_path(p) for p in norm_paths):
+                    expected_list = sorted(expected)
+                    expected_str = ", ".join(expected_list)
+                    return False, (
+                        f"Subject prefix '{subject_prefix}' does not match files changed.\n"
+                        f"Expected one of: {expected_str}"
+                    )
+
         else:
             expected_list = sorted(expected)
             expected_str = ", ".join(expected_list)

.github/scripts/tests/test_commit_lint.py

@@ -311,6 +311,31 @@ def test_error_multiple_prefixes_inferred_from_files():
     assert "does not match files changed" in msg
 
 
+def test_valid_http_server_umbrella_prefix():
+    """
+    Commits touching only the HTTP server interface boundary may use http_server:.
+    """
+    commit = make_commit(
+        "http_server: unify listener startup\n\nSigned-off-by: User",
+        ["src/http_server/flb_hs.c", "src/flb_http_common.c"]
+    )
+    ok, _ = validate_commit(commit)
+    assert ok is True
+
+
+def test_error_http_server_umbrella_with_unrelated_component():
+    """
+    http_server: must not cover unrelated core files outside the HTTP interface.
+    """
+    commit = make_commit(
+        "http_server: adjust startup\n\nSigned-off-by: User",
+        ["src/http_server/flb_hs.c", "src/flb_input.c"]
+    )
+    ok, msg = validate_commit(commit)
+    assert ok is False
+    assert "does not match files changed" in msg
+
+
 
 # -----------------------------------------------------------
 # Edge Cases

include/fluent-bit/flb_http_common.h

@@ -141,6 +141,7 @@ struct flb_http_stream {
 };
 
 struct flb_aws_provider;
+struct flb_connection;
 
 /* HTTP REQUEST */
 
@@ -155,6 +156,8 @@ int flb_http_request_commit(struct flb_http_request *request);
 char *flb_http_request_get_header(struct flb_http_request *request,
                                   char *name);
 
+const char *flb_http_request_get_remote_address(struct flb_http_request *request);
+
 int flb_http_request_set_header(struct flb_http_request *request,
                                 char *name, size_t name_length,
                                 char *value, size_t value_length);
@@ -196,6 +199,8 @@ int flb_http_request_set_body(struct flb_http_request *request,
                               unsigned char *body, size_t body_length,
                               char *compression_algorithm);
 
+int flb_http_request_normalize(struct flb_http_request *request);
+
 int flb_http_request_set_authorization(struct flb_http_request *request,
                                        int type, ...);
 

include/fluent-bit/flb_input.h

@@ -73,12 +73,14 @@
                                      * In addition, if TLS is enabled then a
                                      * private key and certificate are required.
                                      */
+#define FLB_INPUT_HTTP_SERVER 4096  /* input uses the generic HTTP server     */
 
 /* Input status */
 #define FLB_INPUT_RUNNING     1
 #define FLB_INPUT_PAUSED      0
 
 struct flb_input_instance;
+struct flb_http_server_config;
 
 /*
  * Tests callbacks
@@ -162,7 +164,6 @@ struct flb_input_plugin {
     char *description;
 
     struct flb_config_map *config_map;
-
     /* Initialization */
     int (*cb_init)    (struct flb_input_instance *, struct flb_config *, void *);
 
@@ -471,6 +472,10 @@ struct flb_input_instance {
     struct mk_list *net_config_map;
     struct mk_list net_properties;
 
+    struct mk_list *http_server_config_map;
+    struct flb_http_server_config *http_server_config;
+    struct mk_list http_server_properties;
+
     struct mk_list *oauth2_jwt_config_map;
     struct mk_list oauth2_jwt_properties;
 
@@ -742,6 +747,16 @@ static inline int flb_input_config_map_set(struct flb_input_instance *ins,
         }
     }
 
+    /* HTTP server properties */
+    if (ins->http_server_config_map && ins->http_server_config) {
+        ret = flb_config_map_set(&ins->http_server_properties,
+                                 ins->http_server_config_map,
+                                 ins->http_server_config);
+        if (ret == -1) {
+            return -1;
+        }
+    }
+
     return ret;
 }
 

include/fluent-bit/flb_output.h

@@ -83,6 +83,7 @@ int flb_chunk_trace_output(struct flb_chunk_trace *trace, struct flb_output_inst
 #define FLB_OUTPUT_NO_MULTIPLEX  512  /* run one task at a time, one task per flush */
 #define FLB_OUTPUT_PRIVATE      1024
 #define FLB_OUTPUT_SYNCHRONOUS  2048  /* run one task at a time, no flush cycle limit */
+#define FLB_OUTPUT_HTTP_SERVER  4096  /* output uses the generic HTTP server  */
 
 
 /*
@@ -106,6 +107,7 @@ int flb_chunk_trace_output(struct flb_chunk_trace *trace, struct flb_output_inst
     const char *tag    = event_chunk->tag;
 
 struct flb_output_flush;
+struct flb_http_server_config;
 
 /*
  * Tests callbacks
@@ -434,6 +436,10 @@ struct flb_output_instance {
     struct mk_list *net_config_map;
     struct mk_list net_properties;
 
+    struct mk_list *http_server_config_map;
+    struct flb_http_server_config *http_server_config;
+    struct mk_list http_server_properties;
+
     struct mk_list *oauth2_config_map;
     struct mk_list oauth2_properties;
 
@@ -1325,6 +1331,16 @@ static inline int flb_output_config_map_set(struct flb_output_instance *ins,
         }
     }
 
+    /* HTTP server properties */
+    if (ins->http_server_config_map && ins->http_server_config) {
+        ret = flb_config_map_set(&ins->http_server_properties,
+                                 ins->http_server_config_map,
+                                 ins->http_server_config);
+        if (ret == -1) {
+            return -1;
+        }
+    }
+
     /* OAuth2 properties are validated but not automatically applied here.
      * Plugins should call flb_config_map_set() with &ctx->oauth2_config
      * in their init callback after calling flb_output_config_map_set(). */

include/fluent-bit/http_server/flb_hs.h

@@ -22,31 +22,70 @@
 
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_config.h>
+#include <fluent-bit/flb_network.h>
 #include <fluent-bit/flb_sds.h>
-#include <monkey/mk_lib.h>
+#include <fluent-bit/http_server/flb_http_server.h>
+#include <monkey/mk_core.h>
 
 /*
  * HTTP buffers that contains certain cached data to be used
  * by end-points.
  */
 struct flb_hs_buf {
     int users;
+    int pending_free;
     flb_sds_t data;
     void *raw_data;
     size_t raw_size;
     struct mk_list _head;
 };
 
+struct flb_health_check_metrics_counter {
+    int error_limit;
+    int error_counter;
+    int retry_failure_limit;
+    int retry_failure_counter;
+    int period_limit;
+    int period_counter;
+};
+
+struct flb_hs_hc_buf {
+    int users;
+    int error_count;
+    int retry_failure_count;
+    struct mk_list _head;
+};
+
+enum flb_hs_route_match_type {
+    FLB_HS_ROUTE_EXACT = 0,
+    FLB_HS_ROUTE_PREFIX = 1
+};
+
+struct flb_hs;
+
+typedef int (*flb_hs_endpoint_callback)(
+                struct flb_hs *hs,
+                struct flb_http_request *request,
+                struct flb_http_response *response);
+
+struct flb_hs_route {
+    const char *path;
+    int match_type;
+    flb_hs_endpoint_callback callback;
+    struct mk_list _head;
+};
+
 struct flb_hs {
-    mk_ctx_t *ctx;             /* Monkey HTTP Context */
-    int vid;                   /* Virtual Host ID     */
-    int qid_metrics;           /* Metrics Message Queue ID    */
-    int qid_metrics_v2;        /* Metrics Message Queue ID for /api/v2 */
-    int qid_storage;           /* Storage Message Queue ID    */
-    int qid_health;            /* health Message Queue ID    */
+    struct flb_http_server server;
+    struct flb_net_setup net_setup;
+    struct flb_config *config;
+    struct mk_list routes;
+    struct mk_list health_metrics;
+    struct flb_health_check_metrics_counter health_counter;
 
-    pthread_t tid;             /* Server Thread */
-    struct flb_config *config; /* Fluent Bit context */
+    struct flb_hs_buf metrics;
+    struct flb_hs_buf metrics_v2;
+    struct flb_hs_buf storage_metrics;
 
     /* end-point: root */
     size_t ep_root_size;
@@ -62,5 +101,11 @@ int flb_hs_push_storage_metrics(struct flb_hs *hs, void *data, size_t size);
 
 int flb_hs_destroy(struct flb_hs *ctx);
 int flb_hs_start(struct flb_hs *hs);
+void flb_hs_cmt_buffer_destroy(void *data);
+void flb_hs_buf_release(struct flb_hs_buf *buffer, void (*raw_free)(void *));
+int flb_hs_register_endpoint(struct flb_hs *hs,
+                             const char *path,
+                             int match_type,
+                             flb_hs_endpoint_callback callback);
 
 #endif

include/fluent-bit/http_server/flb_hs_utils.h

@@ -21,8 +21,19 @@
 #define FLB_HS_UTIL_H
 
 #include <monkey/mk_lib.h>
+#include <fluent-bit/http_server/flb_hs.h>
 
 int flb_hs_add_content_type_to_req(mk_request_t *request, int type);
+int flb_hs_response_set_content_type(struct flb_http_response *response, int type);
+int flb_hs_response_set_payload(struct flb_http_response *response,
+                                int status,
+                                int type,
+                                const void *payload,
+                                size_t payload_size);
+int flb_hs_response_send_string(struct flb_http_response *response,
+                                int status,
+                                int type,
+                                const char *payload);
 
 /* Content-type */
 enum content_type {